diff --git a/.binder/requirements.txt b/.binder/requirements.txt index 507ff64f7a61e..9ecc5c6fba79c 100644 --- a/.binder/requirements.txt +++ b/.binder/requirements.txt @@ -1,4 +1,4 @@ ---find-links https://sklearn-nightly.scdn8.secure.raxcdn.com scikit-learn +--extra-index https://pypi.anaconda.org/scipy-wheels-nightly/simple scikit-learn --pre matplotlib scikit-image diff --git a/.circleci/config.yml b/.circleci/config.yml index 3933d5404202f..5480831fd962e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,17 +9,17 @@ jobs: - MKL_NUM_THREADS: 2 - MINICONDA_PATH: ~/miniconda - CONDA_ENV_NAME: testenv - - PYTHON_VERSION: 3.5 - - NUMPY_VERSION: 1.11.0 - - SCIPY_VERSION: 0.17.0 - - MATPLOTLIB_VERSION: 1.5.1 + - PYTHON_VERSION: 3.6 + - NUMPY_VERSION: 1.13.3 + - SCIPY_VERSION: 0.19.1 + - MATPLOTLIB_VERSION: 2.1.1 # on conda, this is the latest for python 3.5 # The following places need to be in sync with regard to Cython version: # - .circleci config file # - sklearn/_build_utils/__init__.py # - advanced installation guide - CYTHON_VERSION: 0.28.5 - - SCIKIT_IMAGE_VERSION: 0.12.3 + - SCIKIT_IMAGE_VERSION: 0.13 steps: - checkout - run: ./build_tools/circle/checkout_merge_commit.sh @@ -101,7 +101,7 @@ jobs: pypy3: docker: - - image: pypy:3.6-7.1.1 + - image: pypy:3.6-7.2.0 steps: - restore_cache: keys: diff --git a/.github/ISSUE_TEMPLATE/blank_template.md b/.github/ISSUE_TEMPLATE/blank_template.md new file mode 100644 index 0000000000000..d46ae9e50b18f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/blank_template.md @@ -0,0 +1,10 @@ +--- +name: Other +about: For all other issues to reach the community... +title: '' +labels: '' +assignees: '' + +--- + + diff --git a/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE/bug_report.md similarity index 65% rename from ISSUE_TEMPLATE.md rename to .github/ISSUE_TEMPLATE/bug_report.md index c8ce3e4905b37..102ebd0770535 100644 --- a/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,27 +1,36 @@ +--- +name: Bug report +about: Create a report to help us reproduce and correct the bug +title: '' +labels: 'Bug: triage' +assignees: '' + +--- + - - -#### Description - +#### Describe the bug + #### Steps/Code to Reproduce +``` +Sample code to reproduce the problem +``` + #### Expected Results @@ -51,6 +64,7 @@ import sys; print("Python", sys.version) import numpy; print("NumPy", numpy.__version__) import scipy; print("SciPy", scipy.__version__) import sklearn; print("Scikit-Learn", sklearn.__version__) +import imblearn; print("Imbalanced-Learn", imblearn.__version__) --> diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000000..3ba13e0cec6cb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false diff --git a/.github/ISSUE_TEMPLATE/doc_improvement.md b/.github/ISSUE_TEMPLATE/doc_improvement.md new file mode 100644 index 0000000000000..4c2906bb18418 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/doc_improvement.md @@ -0,0 +1,20 @@ +--- +name: Documentation improvement +about: Create a report to help us improve the documentation. Alternatively you can just open a pull request with the suggested change. +title: '' +labels: Documentation +assignees: '' + +--- + +#### Describe the issue linked to the documentation + + + +#### Suggest a potential alternative/fix + + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000000000..b2ff110d69a04 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,22 @@ +--- +name: Feature request +about: Suggest a new algorithm, enhancement to an existing algorithm, etc. +title: '' +labels: New Feature +assignees: '' + +--- + + + +#### Describe the workflow you want to enable + +#### Describe your proposed solution + +#### Describe alternatives you've considered, if relevant + +#### Additional context diff --git a/.github/ISSUE_TEMPLATE/usage_question.md b/.github/ISSUE_TEMPLATE/usage_question.md new file mode 100644 index 0000000000000..1b0dd8ef8340a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/usage_question.md @@ -0,0 +1,20 @@ +--- +name: Usage question +about: If you have a usage question +title: '' +labels: Question +assignees: '' + +--- + + diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 0000000000000..faf2acdc2e9db --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,80 @@ +module:cluster: +- sklearn/cluster/**/* + +module:common: +- sklearn/common/**/* + +module:compose: +- sklearn/compose/**/* + +module:covariance: +- sklearn/covariance/**/* + +module:cross_decomposition: +- sklearn/cross_decomposition/**/* + +module:datasets: +- sklearn/datasets/**/* + +module:decomposition: +- sklearn/decomposition/**/* + +module:ensemble: +- sklearn/ensemble/**/* + +module:feature_extraction: +- sklearn/feature_extraction/**/* + +module:feature_selection: +- sklearn/feature_selection/**/* + +module:gaussian_process: +- sklearn/gaussian_process/**/* + +module:impute: +- sklearn/impute/**/* + +module:inspection: +- sklearn/inspection/**/* + +module:linear_model: +- sklearn/linear_model/**/* + +module:manifold: +- sklearn/manifold/**/* + +module:metrics: +- sklearn/metrics/**/* + +module:mixture: +- sklearn/mixture/**/* + +module:model_selection: +- sklearn/model_selection/**/* + +module:naive_bayes: +- sklearn/naive_bayes.py + +module:neighbors: +- sklearn/neighbors/**/* + +module:neural_network: +- sklearn/neural_network/**/* + +module:pipeline: +- sklearn/pipeline.py + +module:preprocessing: +- sklearn/preprocessing/**/* + +module:semi_supervised: +- sklearn/semi_supervised/**/* + +module:svm: +- sklearn/svm/**/* + +module:tree: +- sklearn/tree/**/* + +module:utils: +- sklearn/utils/**/* diff --git a/.github/workflows/assign.yml b/.github/workflows/assign.yml new file mode 100644 index 0000000000000..72643d1bf8ae8 --- /dev/null +++ b/.github/workflows/assign.yml @@ -0,0 +1,16 @@ + +name: Assign +on: + issue_comment: + types: created + +jobs: + one: + runs-on: ubuntu-latest + steps: + - if: github.event.comment.body == 'take' + name: + run: | + echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}" + curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees + curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -X "DELETE" https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/labels/help%20wanted diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 0000000000000..28d1debcad7f1 --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,14 @@ +name: "Pull Request Labeler" +on: + schedule: + - cron: "*/10 * * * *" + +jobs: + triage: + runs-on: ubuntu-latest + steps: + - uses: thomasjpfan/labeler@v2.4.6 + if: github.repository == 'scikit-learn/scikit-learn' + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" + max-labels: "3" diff --git a/.github/workflows/twitter.yml b/.github/workflows/twitter.yml new file mode 100644 index 0000000000000..ac2f037246257 --- /dev/null +++ b/.github/workflows/twitter.yml @@ -0,0 +1,26 @@ +# Tweet the URL of a commit on @sklearn_commits whenever a push event +# happens on the master branch +name: Twitter Push Notification + + +on: + push: + branches: + - master + + +jobs: + tweet: + name: Twitter Notification + runs-on: ubuntu-latest + steps: + - name: Tweet URL of last commit as @sklearn_commits + if: github.repository == 'scikit-learn/scikit-learn' + uses: xorilog/twitter-action@0.1 + with: + args: "-message \"https://github.com/scikit-learn/scikit-learn/commit/${{ github.sha }}\"" + env: + TWITTER_CONSUMER_KEY: ${{ secrets.TWITTER_CONSUMER_KEY }} + TWITTER_CONSUMER_SECRET: ${{ secrets.TWITTER_CONSUMER_SECRET }} + TWITTER_ACCESS_TOKEN: ${{ secrets.TWITTER_ACCESS_TOKEN }} + TWITTER_ACCESS_SECRET: ${{ secrets.TWITTER_ACCESS_SECRET }} diff --git a/.github/workflows/unassign.yml b/.github/workflows/unassign.yml new file mode 100644 index 0000000000000..96f1360ba3144 --- /dev/null +++ b/.github/workflows/unassign.yml @@ -0,0 +1,14 @@ +name: Unassign +#Runs when a contributor has unassigned themselves from the issue and adds 'help wanted' and 'stalled' tags +on: + issues: + types: unassigned + +jobs: + one: + runs-on: ubuntu-latest + steps: + - name: + run: | + echo "Marking issue ${{ github.event.issue.number }} as stalled" + curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"labels": ["help wanted","Stalled"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/labels diff --git a/.landscape.yml b/.landscape.yml deleted file mode 100644 index 4774bdc1a2984..0000000000000 --- a/.landscape.yml +++ /dev/null @@ -1,5 +0,0 @@ -pylint: - disable: - - unpacking-non-sequence -ignore-paths: - - sklearn/externals diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000000..f99ec64342af9 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,16 @@ +# Code of Conduct + +We are a community based on openness and friendly, didactic, discussions. + +We aspire to treat everybody equally, and value their contributions. + +Decisions are made based on technical merit and consensus. + +Code is not the only way to help the project. Reviewing pull requests, +answering questions to help others on mailing lists or issues, organizing and +teaching tutorials, working on the website, improving the documentation, are +all priceless contributions. + +We abide by the principles of openness, respect, and consideration of others of +the Python Software Foundation: https://www.python.org/psf/codeofconduct/ + diff --git a/COPYING b/COPYING index 0f665f8400d08..b98af18710185 100644 --- a/COPYING +++ b/COPYING @@ -1,6 +1,6 @@ New BSD License -Copyright (c) 2007–2019 The scikit-learn developers. +Copyright (c) 2007–2020 The scikit-learn developers. All rights reserved. diff --git a/MANIFEST.in b/MANIFEST.in index 04d62596bbf3d..89634452812e4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,3 +5,4 @@ recursive-include sklearn *.c *.h *.pyx *.pxd *.pxi *.tp recursive-include sklearn/datasets *.csv *.csv.gz *.rst *.jpg *.txt *.arff.gz *.json.gz include COPYING include README.rst +include pyproject.toml diff --git a/README.rst b/README.rst index 6e491557ab541..fa0b665bbc8dd 100644 --- a/README.rst +++ b/README.rst @@ -14,8 +14,8 @@ .. |CircleCI| image:: https://circleci.com/gh/scikit-learn/scikit-learn/tree/master.svg?style=shield&circle-token=:circle-token .. _CircleCI: https://circleci.com/gh/scikit-learn/scikit-learn -.. |PythonVersion| image:: https://img.shields.io/pypi/pyversions/scikit-learn.svg -.. _PythonVersion: https://img.shields.io/pypi/pyversions/scikit-learn.svg +.. |PythonVersion| image:: https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue +.. _PythonVersion: https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue .. |PyPi| image:: https://badge.fury.io/py/scikit-learn.svg .. _PyPi: https://badge.fury.io/py/scikit-learn @@ -31,7 +31,7 @@ SciPy and is distributed under the 3-Clause BSD license. The project was started in 2007 by David Cournapeau as a Google Summer of Code project, and since then many volunteers have contributed. See -the `About us `_ page +the `About us `__ page for a list of core contributors. It is currently maintained by a team of volunteers. @@ -47,18 +47,18 @@ Dependencies scikit-learn requires: -- Python (>= 3.5) -- NumPy (>= 1.11.0) -- SciPy (>= 0.17.0) +- Python (>= 3.6) +- NumPy (>= 1.13.3) +- SciPy (>= 0.19.1) - joblib (>= 0.11) **Scikit-learn 0.20 was the last version to support Python 2.7 and Python 3.4.** -scikit-learn 0.21 and later require Python 3.5 or newer. +scikit-learn 0.23 and later require Python 3.6 or newer. -Scikit-learn plotting capabilities (i.e., functions start with "plot_" -and classes end with "Display") require Matplotlib (>= 1.5.1). For running the -examples Matplotlib >= 1.5.1 is required. A few examples require -scikit-image >= 0.12.3, a few examples require pandas >= 0.18.0. +Scikit-learn plotting capabilities (i.e., functions start with ``plot_`` +and classes end with "Display") require Matplotlib (>= 2.1.1). For running the +examples Matplotlib >= 2.1.1 is required. A few examples require +scikit-image >= 0.13, a few examples require pandas >= 0.18.0. User installation ~~~~~~~~~~~~~~~~~ @@ -138,7 +138,7 @@ Project History The project was started in 2007 by David Cournapeau as a Google Summer of Code project, and since then many volunteers have contributed. See -the `About us `_ page +the `About us `__ page for a list of core contributors. The project is currently maintained by a team of volunteers. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9fdead91dd309..1aad015849b2e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,9 +1,17 @@ # Adapted from https://github.com/pandas-dev/pandas/blob/master/azure-pipelines.yml +schedules: +- cron: "30 2 * * *" + displayName: Run nightly build + branches: + include: + - master + always: true + jobs: - job: linting displayName: Linting pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 steps: - bash: echo "##vso[task.prependpath]$CONDA/bin" displayName: Add conda to PATH @@ -12,63 +20,98 @@ jobs: - bash: conda create --name flake8_env --yes flake8 displayName: Install flake8 - bash: | - source activate flake8_env - ./build_tools/circle/linting.sh + if [[ $BUILD_SOURCEVERSIONMESSAGE =~ \[lint\ skip\] ]]; then + # skip linting + echo "Skipping linting" + exit 0 + else + source activate flake8_env + ./build_tools/circle/linting.sh + fi displayName: Run linting + - bash: | + if [[ $BUILD_SOURCEVERSIONMESSAGE =~ \[scipy-dev\] ]] || \ + [[ $BUILD_REASON == "Schedule" ]]; then + echo "##vso[task.setvariable variable=runScipyDev;isOutput=true]true" + else + echo "##vso[task.setvariable variable=runScipyDev;isOutput=true]false" + fi + name: gitCommitMessage + displayName: Determine to run scipy-dev +- template: build_tools/azure/posix.yml + parameters: + name: Linux_Nightly + vmImage: ubuntu-18.04 + dependsOn: [linting] + condition: eq(dependencies['linting']['outputs']['gitCommitMessage.runScipyDev'], 'true') + matrix: + pylatest_pip_scipy_dev: + DISTRIB: 'conda-pip-scipy-dev' + PYTHON_VERSION: '*' + CHECK_WARNINGS: 'true' + CHECK_PYTEST_SOFT_DEPENDENCY: 'true' + TEST_DOCSTRINGS: 'true' + # Tests that require large downloads over the networks are skipped in CI. + # Here we make sure, that they are still run on a regular basis. + SKLEARN_SKIP_NETWORK_TESTS: '0' # Will run all the time regardless of linting outcome. - template: build_tools/azure/posix.yml parameters: name: Linux_Runs - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 matrix: pylatest_conda_mkl: DISTRIB: 'conda' PYTHON_VERSION: '*' - INSTALL_MKL: 'true' + BLAS: 'mkl' NUMPY_VERSION: '*' SCIPY_VERSION: '*' CYTHON_VERSION: '*' PILLOW_VERSION: '*' PYTEST_VERSION: '*' JOBLIB_VERSION: '*' + THREADPOOLCTL_VERSION: '2.0.0' COVERAGE: 'true' - template: build_tools/azure/posix.yml parameters: name: Linux - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 dependsOn: [linting] matrix: # Linux environment to test that scikit-learn can be built against - # versions of numpy, scipy with ATLAS that comes with Ubuntu Xenial 16.04 - # i.e. numpy 1.11 and scipy 0.17 - py35_ubuntu_atlas: + # versions of numpy, scipy with ATLAS that comes with Ubuntu Bionic 18.04 + # i.e. numpy 1.13.3 and scipy 0.19 + py36_ubuntu_atlas: DISTRIB: 'ubuntu' - PYTHON_VERSION: '3.5' + PYTHON_VERSION: '3.6' JOBLIB_VERSION: '0.11' - # Linux + Python 3.5 build with OpenBLAS and without SITE_JOBLIB - py35_conda_openblas: + THREADPOOLCTL_VERSION: '2.0.0' + # Linux + Python 3.6 build with OpenBLAS and without SITE_JOBLIB + py36_conda_openblas: DISTRIB: 'conda' - PYTHON_VERSION: '3.5' - INSTALL_MKL: 'false' - NUMPY_VERSION: '1.11.0' - SCIPY_VERSION: '0.17.0' + PYTHON_VERSION: '3.6' + BLAS: 'openblas' + NUMPY_VERSION: '1.13.3' + SCIPY_VERSION: '0.19.1' PANDAS_VERSION: '*' CYTHON_VERSION: '*' - PYTEST_VERSION: '*' - PILLOW_VERSION: '4.0.0' - MATPLOTLIB_VERSION: '1.5.1' - # later version of joblib are not packaged in conda for Python 3.5 - JOBLIB_VERSION: '0.12.3' + # temporary pin pytest due to unknown failure with pytest 5.3 + PYTEST_VERSION: '5.2' + PILLOW_VERSION: '4.2.1' + MATPLOTLIB_VERSION: '2.1.1' + SCIKIT_IMAGE_VERSION: '*' + # latest version of joblib available in conda for Python 3.6 + JOBLIB_VERSION: '0.13.2' + THREADPOOLCTL_VERSION: '2.0.0' COVERAGE: 'true' # Linux environment to test the latest available dependencies and MKL. - # It runs tests requiring pandas and PyAMG. + # It runs tests requiring lightgbm, pandas and PyAMG. pylatest_pip_openblas_pandas: DISTRIB: 'conda-pip-latest' - # FIXME: pinned until SciPy wheels are available for Pyhon 3.8 - PYTHON_VERSION: '3.7' + PYTHON_VERSION: '3.8' PYTEST_VERSION: '4.6.2' COVERAGE: 'true' CHECK_PYTEST_SOFT_DEPENDENCY: 'true' @@ -78,41 +121,44 @@ jobs: - template: build_tools/azure/posix-32.yml parameters: name: Linux32 - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 dependsOn: [linting] matrix: - py35_ubuntu_atlas_32bit: + py36_ubuntu_atlas_32bit: DISTRIB: 'ubuntu-32' - PYTHON_VERSION: '3.5' - JOBLIB_VERSION: '0.11' + PYTHON_VERSION: '3.6' + JOBLIB_VERSION: '0.13' + THREADPOOLCTL_VERSION: '2.0.0' - template: build_tools/azure/posix.yml parameters: name: macOS - vmImage: xcode9-macos10.13 + vmImage: macOS-10.14 dependsOn: [linting] matrix: pylatest_conda_mkl: DISTRIB: 'conda' PYTHON_VERSION: '*' - INSTALL_MKL: 'true' + BLAS: 'mkl' NUMPY_VERSION: '*' SCIPY_VERSION: '*' CYTHON_VERSION: '*' PILLOW_VERSION: '*' PYTEST_VERSION: '*' JOBLIB_VERSION: '*' + THREADPOOLCTL_VERSION: '2.0.0' COVERAGE: 'true' pylatest_conda_mkl_no_openmp: DISTRIB: 'conda' PYTHON_VERSION: '*' - INSTALL_MKL: 'true' + BLAS: 'mkl' NUMPY_VERSION: '*' SCIPY_VERSION: '*' CYTHON_VERSION: '*' PILLOW_VERSION: '*' PYTEST_VERSION: '*' JOBLIB_VERSION: '*' + THREADPOOLCTL_VERSION: '2.0.0' COVERAGE: 'true' SKLEARN_TEST_NO_OPENMP: 'true' SKLEARN_SKIP_OPENMP_TEST: 'true' @@ -129,6 +175,6 @@ jobs: PYTHON_ARCH: '64' PYTEST_VERSION: '*' COVERAGE: 'true' - py35_pip_openblas_32bit: - PYTHON_VERSION: '3.5' + py36_pip_openblas_32bit: + PYTHON_VERSION: '3.6' PYTHON_ARCH: '32' diff --git a/benchmarks/bench_hist_gradient_boosting.py b/benchmarks/bench_hist_gradient_boosting.py index 9bfd6d743ee4f..ac7e77e2a1f99 100644 --- a/benchmarks/bench_hist_gradient_boosting.py +++ b/benchmarks/bench_hist_gradient_boosting.py @@ -32,6 +32,9 @@ parser.add_argument('--n-samples-max', type=int, default=int(1e6)) parser.add_argument('--n-features', type=int, default=20) parser.add_argument('--max-bins', type=int, default=255) +parser.add_argument('--random-sample-weights', action="store_true", + default=False, + help="generate and use random sample weights") args = parser.parse_args() n_leaf_nodes = args.n_leaf_nodes @@ -46,6 +49,7 @@ def get_estimator_and_data(): n_features=args.n_features, n_classes=args.n_classes, n_clusters_per_class=1, + n_informative=args.n_classes, random_state=0) return X, y, HistGradientBoostingClassifier elif args.problem == 'regression': @@ -60,8 +64,19 @@ def get_estimator_and_data(): np.bool) X[mask] = np.nan -X_train_, X_test_, y_train_, y_test_ = train_test_split( - X, y, test_size=0.5, random_state=0) +if args.random_sample_weights: + sample_weight = np.random.rand(len(X)) * 10 +else: + sample_weight = None + +if sample_weight is not None: + (X_train_, X_test_, y_train_, y_test_, + sample_weight_train_, _) = train_test_split( + X, y, sample_weight, test_size=0.5, random_state=0) +else: + X_train_, X_test_, y_train_, y_test_ = train_test_split( + X, y, test_size=0.5, random_state=0) + sample_weight_train_ = None def one_run(n_samples): @@ -69,6 +84,10 @@ def one_run(n_samples): X_test = X_test_[:n_samples] y_train = y_train_[:n_samples] y_test = y_test_[:n_samples] + if sample_weight is not None: + sample_weight_train = sample_weight_train_[:n_samples] + else: + sample_weight_train = None assert X_train.shape[0] == n_samples assert X_test.shape[0] == n_samples print("Data size: %d samples train, %d samples test." @@ -79,7 +98,7 @@ def one_run(n_samples): max_iter=n_trees, max_bins=max_bins, max_leaf_nodes=n_leaf_nodes, - n_iter_no_change=None, + early_stopping=False, random_state=0, verbose=0) loss = args.loss @@ -93,7 +112,7 @@ def one_run(n_samples): if loss == 'default': loss = 'least_squares' est.set_params(loss=loss) - est.fit(X_train, y_train) + est.fit(X_train, y_train, sample_weight=sample_weight_train) sklearn_fit_duration = time() - tic tic = time() sklearn_score = est.score(X_test, y_test) @@ -110,7 +129,7 @@ def one_run(n_samples): lightgbm_est = get_equivalent_estimator(est, lib='lightgbm') tic = time() - lightgbm_est.fit(X_train, y_train) + lightgbm_est.fit(X_train, y_train, sample_weight=sample_weight_train) lightgbm_fit_duration = time() - tic tic = time() lightgbm_score = lightgbm_est.score(X_test, y_test) @@ -127,7 +146,7 @@ def one_run(n_samples): xgb_est = get_equivalent_estimator(est, lib='xgboost') tic = time() - xgb_est.fit(X_train, y_train) + xgb_est.fit(X_train, y_train, sample_weight=sample_weight_train) xgb_fit_duration = time() - tic tic = time() xgb_score = xgb_est.score(X_test, y_test) @@ -144,7 +163,7 @@ def one_run(n_samples): cat_est = get_equivalent_estimator(est, lib='catboost') tic = time() - cat_est.fit(X_train, y_train) + cat_est.fit(X_train, y_train, sample_weight=sample_weight_train) cat_fit_duration = time() - tic tic = time() cat_score = cat_est.score(X_test, y_test) diff --git a/benchmarks/bench_plot_hierarchical.py b/benchmarks/bench_plot_hierarchical.py new file mode 100644 index 0000000000000..3c8cd4464a771 --- /dev/null +++ b/benchmarks/bench_plot_hierarchical.py @@ -0,0 +1,85 @@ +from collections import defaultdict +from time import time + +import numpy as np +from numpy import random as nr + +from sklearn.cluster import AgglomerativeClustering + + +def compute_bench(samples_range, features_range): + + it = 0 + results = defaultdict(lambda: []) + + max_it = len(samples_range) * len(features_range) + for n_samples in samples_range: + for n_features in features_range: + it += 1 + print('==============================') + print('Iteration %03d of %03d' % (it, max_it)) + print('n_samples %05d; n_features %02d' % (n_samples, n_features)) + print('==============================') + print() + data = nr.randint(-50, 51, (n_samples, n_features)) + + for linkage in ("single", "average", "complete", "ward"): + print(linkage.capitalize()) + tstart = time() + AgglomerativeClustering( + linkage=linkage, + n_clusters=10 + ).fit(data) + + delta = time() - tstart + print("Speed: %0.3fs" % delta) + print() + + results[linkage].append(delta) + + return results + + +if __name__ == '__main__': + import matplotlib.pyplot as plt + + samples_range = np.linspace(1000, 15000, 8).astype(np.int) + features_range = np.array([2, 10, 20, 50]) + + results = compute_bench(samples_range, features_range) + + max_time = max([max(i) for i in [t for (label, t) in results.items()]]) + + colors = plt.get_cmap('tab10')(np.linspace(0, 1, 10))[:4] + lines = {linkage: None for linkage in results.keys()} + fig, axs = plt.subplots(2, 2, sharex=True, sharey=True) + fig.suptitle( + 'Scikit-learn agglomerative clustering benchmark results', + fontsize=16 + ) + for c, (label, timings) in zip(colors, + sorted(results.items())): + timing_by_samples = np.asarray(timings).reshape( + samples_range.shape[0], + features_range.shape[0] + ) + + for n in range(timing_by_samples.shape[1]): + ax = axs.flatten()[n] + lines[label], = ax.plot( + samples_range, + timing_by_samples[:, n], + color=c, + label=label + ) + ax.set_title('n_features = %d' % features_range[n]) + if n >= 2: + ax.set_xlabel('n_samples') + if n % 2 == 0: + ax.set_ylabel('time (s)') + + fig.subplots_adjust(right=0.8) + fig.legend([lines[link] for link in sorted(results.keys())], + sorted(results.keys()), loc="center right", fontsize=8) + + plt.show() diff --git a/benchmarks/bench_plot_randomized_svd.py b/benchmarks/bench_plot_randomized_svd.py index 0fe050eaf7e30..e322cda8e87e9 100644 --- a/benchmarks/bench_plot_randomized_svd.py +++ b/benchmarks/bench_plot_randomized_svd.py @@ -104,7 +104,7 @@ # in case the reconstructed (dense) matrix is too large MAX_MEMORY = np.int(2e9) -# The following datasets can be dowloaded manually from: +# The following datasets can be downloaded manually from: # CIFAR 10: https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz # SVHN: http://ufldl.stanford.edu/housenumbers/train_32x32.mat CIFAR_FOLDER = "./cifar-10-batches-py/" diff --git a/benchmarks/bench_text_vectorizers.py b/benchmarks/bench_text_vectorizers.py index 196e677e9b49c..96dbc04312291 100644 --- a/benchmarks/bench_text_vectorizers.py +++ b/benchmarks/bench_text_vectorizers.py @@ -32,7 +32,7 @@ def f(): text = fetch_20newsgroups(subset='train').data[:1000] print("="*80 + '\n#' + " Text vectorizers benchmark" + '\n' + '='*80 + '\n') -print("Using a subset of the 20 newsrgoups dataset ({} documents)." +print("Using a subset of the 20 newsgroups dataset ({} documents)." .format(len(text))) print("This benchmarks runs in ~1 min ...") diff --git a/build_tools/azure/install.cmd b/build_tools/azure/install.cmd index 2566ba4f4f3aa..aa32e7cf2612d 100644 --- a/build_tools/azure/install.cmd +++ b/build_tools/azure/install.cmd @@ -15,6 +15,8 @@ IF "%PYTHON_ARCH%"=="64" ( call activate %VIRTUALENV% + pip install threadpoolctl + IF "%PYTEST_VERSION%"=="*" ( pip install pytest ) else ( @@ -22,7 +24,7 @@ IF "%PYTHON_ARCH%"=="64" ( ) pip install pytest-xdist ) else ( - pip install numpy scipy cython pytest wheel pillow joblib + pip install numpy scipy cython pytest wheel pillow joblib threadpoolctl ) if "%COVERAGE%" == "true" ( pip install coverage codecov pytest-cov diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh index 61ee6bac7116f..0a3ca4e034efd 100755 --- a/build_tools/azure/install.sh +++ b/build_tools/azure/install.sh @@ -1,6 +1,7 @@ #!/bin/bash set -e +set -x UNAMESTR=`uname` @@ -11,9 +12,9 @@ make_conda() { } version_ge() { - # The two version numbers are seperated with a new line is piped to sort + # The two version numbers are separated with a new line is piped to sort # -rV. The -V activates for version number sorting and -r sorts in - # decending order. If the first argument is the top element of the sort, it + # descending order. If the first argument is the top element of the sort, it # is greater than or equal to the second argument. test "$(printf "${1}\n${2}" | sort -rV | head -n 1)" == "$1" } @@ -22,13 +23,8 @@ if [[ "$DISTRIB" == "conda" ]]; then TO_INSTALL="python=$PYTHON_VERSION pip \ numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION \ - cython=$CYTHON_VERSION joblib=$JOBLIB_VERSION" - - if [[ "$INSTALL_MKL" == "true" ]]; then - TO_INSTALL="$TO_INSTALL mkl" - else - TO_INSTALL="$TO_INSTALL nomkl" - fi + cython=$CYTHON_VERSION joblib=$JOBLIB_VERSION\ + blas[build=$BLAS]" if [[ -n "$PANDAS_VERSION" ]]; then TO_INSTALL="$TO_INSTALL pandas=$PANDAS_VERSION" @@ -42,6 +38,10 @@ if [[ "$DISTRIB" == "conda" ]]; then TO_INSTALL="$TO_INSTALL pillow=$PILLOW_VERSION" fi + if [[ -n "$SCIKIT_IMAGE_VERSION" ]]; then + TO_INSTALL="$TO_INSTALL scikit-image=$SCIKIT_IMAGE_VERSION" + fi + if [[ -n "$MATPLOTLIB_VERSION" ]]; then TO_INSTALL="$TO_INSTALL matplotlib=$MATPLOTLIB_VERSION" fi @@ -49,7 +49,7 @@ if [[ "$DISTRIB" == "conda" ]]; then if [[ "$UNAMESTR" == "Darwin" ]]; then if [[ "$SKLEARN_TEST_NO_OPENMP" != "true" ]]; then # on macOS, install an OpenMP-enabled clang/llvm from conda-forge. - TO_INSTALL="$TO_INSTALL conda-forge::compilers \ + TO_INSTALL="$TO_INSTALL conda-forge::compilers>=1.0.4 \ conda-forge::llvm-openmp" fi fi @@ -65,6 +65,8 @@ if [[ "$DISTRIB" == "conda" ]]; then make_conda $TO_INSTALL + pip install threadpoolctl==$THREADPOOLCTL_VERSION + if [[ "$PYTEST_VERSION" == "*" ]]; then python -m pip install pytest else @@ -78,25 +80,37 @@ if [[ "$DISTRIB" == "conda" ]]; then elif [[ "$DISTRIB" == "ubuntu" ]]; then sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test sudo apt-get update - sudo apt-get install python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv + sudo apt-get install python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev python3-virtualenv python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV source $VIRTUALENV/bin/activate - python -m pip install pytest==$PYTEST_VERSION pytest-cov cython joblib==$JOBLIB_VERSION + python -m pip install pytest==$PYTEST_VERSION pytest-cov cython joblib==$JOBLIB_VERSION threadpoolctl==$THREADPOOLCTL_VERSION elif [[ "$DISTRIB" == "ubuntu-32" ]]; then apt-get update - apt-get install -y python3-dev python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv + apt-get install -y python3-dev python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev python3-virtualenv python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV source $VIRTUALENV/bin/activate - python -m pip install pytest==$PYTEST_VERSION pytest-cov cython joblib==$JOBLIB_VERSION + python -m pip install pytest==$PYTEST_VERSION pytest-cov cython joblib==$JOBLIB_VERSION threadpoolctl==$THREADPOOLCTL_VERSION elif [[ "$DISTRIB" == "conda-pip-latest" ]]; then # Since conda main channel usually lacks behind on the latest releases, # we use pypi to test against the latest releases of the dependencies. # conda is still used as a convenient way to install Python and pip. make_conda "python=$PYTHON_VERSION" python -m pip install -U pip - python -m pip install numpy scipy cython joblib python -m pip install pytest==$PYTEST_VERSION pytest-cov pytest-xdist - python -m pip install pandas matplotlib pyamg + python -m pip install pandas matplotlib pyamg scikit-image + # do not install dependencies for lightgbm since it requires scikit-learn + python -m pip install lightgbm --no-deps +elif [[ "$DISTRIB" == "conda-pip-scipy-dev" ]]; then + make_conda "python=$PYTHON_VERSION" + python -m pip install -U pip + python -m pip install pytest==$PYTEST_VERSION pytest-cov pytest-xdist + echo "Installing numpy and scipy master wheels" + dev_url=https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com + pip install --pre --upgrade --timeout=60 -f $dev_url numpy scipy pandas cython + echo "Installing joblib master" + pip install https://github.com/joblib/joblib/archive/master.zip + echo "Installing pillow master" + pip install https://github.com/python-pillow/Pillow/archive/master.zip fi if [[ "$COVERAGE" == "true" ]]; then @@ -123,7 +137,15 @@ except ImportError: " python -m pip list -# Use setup.py instead of `pip install -e .` to be able to pass the -j flag -# to speed-up the building multicore CI machines. -python setup.py build_ext --inplace -j 3 -python setup.py develop +if [[ "$DISTRIB" == "conda-pip-latest" ]]; then + # Check that pip can automatically install the build dependencies from + # pyproject.toml using an isolated build environment: + pip install --verbose --editable . +else + # Use the pre-installed build dependencies and build directly in the + # current environment. + # Use setup.py instead of `pip install -e .` to be able to pass the -j flag + # to speed-up the building multicore CI machines. + python setup.py build_ext --inplace -j 3 + python setup.py develop +fi diff --git a/build_tools/azure/posix-32.yml b/build_tools/azure/posix-32.yml index 68e05e347f307..d6ad049a2376d 100644 --- a/build_tools/azure/posix-32.yml +++ b/build_tools/azure/posix-32.yml @@ -3,10 +3,12 @@ parameters: vmImage: '' matrix: [] dependsOn: [] + condition: ne(variables['Build.Reason'], 'Schedule') jobs: - job: ${{ parameters.name }} dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} pool: vmImage: ${{ parameters.vmImage }} variables: @@ -36,11 +38,12 @@ jobs: -e JUNITXML=$JUNITXML -e VIRTUALENV=testvenv -e JOBLIB_VERSION=$JOBLIB_VERSION + -e THREADPOOLCTL_VERSION=$THREADPOOLCTL_VERSION -e PYTEST_VERSION=$PYTEST_VERSION -e OMP_NUM_THREADS=$OMP_NUM_THREADS -e OPENBLAS_NUM_THREADS=$OPENBLAS_NUM_THREADS -e SKLEARN_SKIP_NETWORK_TESTS=$SKLEARN_SKIP_NETWORK_TESTS - i386/ubuntu:16.04 + i386/ubuntu:18.04 sleep 1000000 displayName: 'Start container' - script: > diff --git a/build_tools/azure/posix.yml b/build_tools/azure/posix.yml index f5c4a023b4c39..9efb0418278d2 100644 --- a/build_tools/azure/posix.yml +++ b/build_tools/azure/posix.yml @@ -3,10 +3,12 @@ parameters: vmImage: '' matrix: [] dependsOn: [] + condition: ne(variables['Build.Reason'], 'Schedule') jobs: - job: ${{ parameters.name }} dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} pool: vmImage: ${{ parameters.vmImage }} variables: @@ -40,7 +42,7 @@ jobs: - script: | build_tools/azure/test_pytest_soft_dependency.sh displayName: 'Test Soft Dependency' - condition: and(eq(variables['CHECK_PYTEST_SOFT_DEPENDENCY'], 'true'), eq(variables['DISTRIB'], 'conda')) + condition: eq(variables['CHECK_PYTEST_SOFT_DEPENDENCY'], 'true') - task: PublishTestResults@2 inputs: testResultsFiles: '$(TEST_DIR)/$(JUNITXML)' @@ -49,7 +51,7 @@ jobs: condition: succeededOrFailed() - script: | build_tools/azure/upload_codecov.sh - condition: and(succeeded(), eq(variables['COVERAGE'], 'true'), eq(variables['DISTRIB'], 'conda')) + condition: and(succeeded(), eq(variables['COVERAGE'], 'true')) displayName: 'Upload To Codecov' env: CODECOV_TOKEN: $(CODECOV_TOKEN) diff --git a/build_tools/azure/windows.yml b/build_tools/azure/windows.yml index 24b542b227dd8..e449eb0f993d0 100644 --- a/build_tools/azure/windows.yml +++ b/build_tools/azure/windows.yml @@ -4,10 +4,12 @@ parameters: vmImage: '' matrix: [] dependsOn: [] + condition: ne(variables['Build.Reason'], 'Schedule') jobs: - job: ${{ parameters.name }} dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} pool: vmImage: ${{ parameters.vmImage }} variables: diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index a76f9a8a890c4..abc823facee15 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -58,6 +58,44 @@ get_build_type() { return fi changed_examples=$(echo "$filenames" | grep -E "^examples/(.*/)*plot_") + + # The following is used to extract the list of filenames of example python + # files that sphinx-gallery needs to run to generate png files used as + # figures or images in the .rst files from the documentation. + # If the contributor changes a .rst file in a PR we need to run all + # the examples mentioned in that file to get sphinx build the + # documentation without generating spurious warnings related to missing + # png files. + + if [[ -n "$filenames" ]] + then + # get rst files + rst_files="$(echo "$filenames" | grep -E "rst$")" + + # get lines with figure or images + img_fig_lines="$(echo "$rst_files" | xargs grep -shE "(figure|image)::")" + + # get only auto_examples + auto_example_files="$(echo "$img_fig_lines" | grep auto_examples | awk -F "/" '{print $NF}')" + + # remove "sphx_glr_" from path and accept replace _(\d\d\d|thumb).png with .py + scripts_names="$(echo "$auto_example_files" | sed 's/sphx_glr_//' | sed -E 's/_([[:digit:]][[:digit:]][[:digit:]]|thumb).png/.py/')" + + # get unique values + examples_in_rst="$(echo "$scripts_names" | uniq )" + fi + + # executed only if there are examples in the modified rst files + if [[ -n "$examples_in_rst" ]] + then + if [[ -n "$changed_examples" ]] + then + changed_examples="$changed_examples|$examples_in_rst" + else + changed_examples="$examples_in_rst" + fi + fi + if [[ -n "$changed_examples" ]] then echo BUILD: detected examples/ filename modified in $git_range: $changed_examples @@ -134,8 +172,8 @@ conda create -n $CONDA_ENV_NAME --yes --quiet python="${PYTHON_VERSION:-*}" \ joblib memory_profiler packaging source activate testenv -pip install sphinx-gallery==0.3.1 -pip install numpydoc==0.9 +pip install sphinx-gallery +pip install numpydoc # Build and install scikit-learn in dev mode python setup.py build_ext --inplace -j 3 @@ -204,5 +242,12 @@ then echo "$warnings" | sed 's/\/home\/circleci\/project\//
  • /g' echo '' ) > 'doc/_build/html/stable/_changed.html' + + if [ "$warnings" != "/home/circleci/project/ no warnings" ] + then + echo "Sphinx generated warnings when building the documentation related to files modified in this PR." + echo "Please check doc/_build/html/stable/_changed.html" + exit 1 + fi fi diff --git a/build_tools/circle/build_test_pypy.sh b/build_tools/circle/build_test_pypy.sh index 60b81e60709f0..dd7cdf3a93654 100755 --- a/build_tools/circle/build_test_pypy.sh +++ b/build_tools/circle/build_test_pypy.sh @@ -3,7 +3,7 @@ set -x set -e apt-get -yq update -apt-get -yq install libatlas-dev libatlas-base-dev liblapack-dev gfortran ccache libopenblas-dev +apt-get -yq install libatlas-base-dev liblapack-dev gfortran ccache libopenblas-dev pip install virtualenv @@ -18,11 +18,14 @@ source pypy-env/bin/activate python --version which python -# XXX: numpy version pinning can be reverted once PyPy -# compatibility is resolved for numpy v1.6.x. For instance, -# when PyPy3 >6.0 is released (see numpy/numpy#12740) -pip install --extra-index https://antocuni.github.io/pypy-wheels/ubuntu numpy Cython pytest -pip install scipy sphinx numpydoc docutils joblib pillow +pip install -U pip + +# pins versions to install wheel from https://antocuni.github.io/pypy-wheels/manylinux2010 +pip install --extra-index-url https://antocuni.github.io/pypy-wheels/manylinux2010 numpy==1.18.0 scipy==1.3.2 + +# Install Cython directly +pip install https://antocuni.github.io/pypy-wheels/ubuntu/Cython/Cython-0.29.14-py3-none-any.whl +pip install sphinx numpydoc docutils joblib pillow pytest ccache -M 512M export CCACHE_COMPRESS=1 @@ -31,7 +34,7 @@ export LOKY_MAX_CPU_COUNT="2" export OMP_NUM_THREADS="1" python setup.py build_ext --inplace -j 3 -pip install -e . +pip install --no-build-isolation -e . # Check that Python implementation is PyPy python - << EOL diff --git a/build_tools/travis/test_script.sh b/build_tools/travis/test_script.sh index f13e0f1bbb2fa..a9c8fb73f9552 100755 --- a/build_tools/travis/test_script.sh +++ b/build_tools/travis/test_script.sh @@ -30,10 +30,9 @@ run_tests() { cp setup.cfg $TEST_DIR cd $TEST_DIR - # Skip tests that require large downloads over the network to save bandwidth - # usage as travis workers are stateless and therefore traditional local - # disk caching does not work. - export SKLEARN_SKIP_NETWORK_TESTS=1 + # Tests that require large downloads over the networks are skipped in CI. + # Here we make sure, that they are still run on a regular basis. + export SKLEARN_SKIP_NETWORK_TESTS=0 if [[ "$COVERAGE" == "true" ]]; then TEST_CMD="$TEST_CMD --cov sklearn" diff --git a/conftest.py b/conftest.py index f640c0e3d001f..17c3f4b144346 100644 --- a/conftest.py +++ b/conftest.py @@ -6,6 +6,7 @@ # the one from site-packages. import platform +import sys from distutils.version import LooseVersion import os @@ -37,7 +38,7 @@ def pytest_collection_modifyitems(config, items): skip_marker = pytest.mark.skip( reason='FeatureHasher is not compatible with PyPy') for item in items: - if item.name.endswith(('hashing.FeatureHasher', + if item.name.endswith(('_hash.FeatureHasher', 'text.HashingVectorizer')): item.add_marker(skip_marker) @@ -61,6 +62,10 @@ def pytest_collection_modifyitems(config, items): reason = ('doctest are only run when the default numpy int is ' '64 bits.') skip_doctests = True + elif sys.platform.startswith("win32"): + reason = ("doctests are not run for Windows because numpy arrays " + "repr is inconsistent across platforms.") + skip_doctests = True except ImportError: pass diff --git a/doc/Makefile b/doc/Makefile index 11c5d58749bec..1cbce7dba9662 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -SPHINXOPTS = +SPHINXOPTS = -j auto SPHINXBUILD ?= sphinx-build PAPER = BUILDDIR = _build diff --git a/doc/about.rst b/doc/about.rst index 2008d96af0045..9926f62dcc824 100644 --- a/doc/about.rst +++ b/doc/about.rst @@ -139,7 +139,7 @@ Grisel, Guillaume Lemaitre, Jérémie du Boisberranger and Chiara Marmo. :target: https://www.axa.fr/ .. |bnp| image:: images/bnp.png - :width: 170pt + :width: 150pt :target: https://www.bnpparibascardif.com/ .. |fujitsu| image:: images/fujitsu.png @@ -175,15 +175,17 @@ Grisel, Guillaume Lemaitre, Jérémie du Boisberranger and Chiara Marmo. +---------+----------+ | | +---------+----------+ - | |axa| ||fujitsu| | + | |axa| | |bnp| | +---------+----------+ - | |bnp| | + ||fujitsu|| |intel| | +---------+----------+ - | |intel| | |nvidia| | + | | + +---------+----------+ + ||dataiku|| |nvidia| | +---------+----------+ | | +---------+----------+ - ||dataiku|| |inria| | + | |inria| | +---------+----------+ .. raw:: html diff --git a/doc/conf.py b/doc/conf.py index 7959a0862f547..d8350a9713ebd 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -51,6 +51,7 @@ if os.environ.get('NO_MATHJAX'): extensions.append('sphinx.ext.imgmath') imgmath_image_format = 'svg' + mathjax_path = '' else: extensions.append('sphinx.ext.mathjax') mathjax_path = ('https://cdn.jsdelivr.net/npm/mathjax@3/es5/' @@ -86,9 +87,14 @@ # # The short X.Y version. import sklearn -version = parse(sklearn.__version__).base_version +parsed_version = parse(sklearn.__version__) +version = ".".join(parsed_version.base_version.split(".")[:2]) # The full version, including alpha/beta/rc tags. -release = sklearn.__version__ +# Removes post from release name +if parsed_version.is_postrelease: + release = parsed_version.base_version +else: + release = sklearn.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -291,7 +297,7 @@ def __call__(self, directory): sphinx_gallery_conf = { 'doc_module': 'sklearn', 'backreferences_dir': os.path.join('modules', 'generated'), - 'show_memory': True, + 'show_memory': False, 'reference_url': { 'sklearn': None}, 'examples_dirs': ['../examples'], @@ -304,7 +310,9 @@ def __call__(self, directory): 'branch': binder_branch, 'dependencies': './binder/requirements.txt', 'use_jupyter_lab': True - } + }, + # avoid generating too many cross links + 'inspect_global_variables': False, } diff --git a/doc/conftest.py b/doc/conftest.py index d1be865135e76..eacd469f2e52f 100644 --- a/doc/conftest.py +++ b/doc/conftest.py @@ -58,6 +58,11 @@ def setup_impute(): def setup_unsupervised_learning(): + try: + import skimage # noqa + except ImportError: + raise SkipTest("Skipping unsupervised_learning.rst, scikit-image " + "not installed") # ignore deprecation warnings from scipy.misc.face warnings.filterwarnings('ignore', 'The binary mode of fromstring', DeprecationWarning) diff --git a/doc/datasets/index.rst b/doc/datasets/index.rst index 2fb7e84610833..88ae88d7a3151 100644 --- a/doc/datasets/index.rst +++ b/doc/datasets/index.rst @@ -21,46 +21,50 @@ also possible to generate synthetic data. General dataset API =================== -There are three main kinds of dataset interfaces that can be used to get +There are three main kinds of dataset interfaces that can be used to get datasets depending on the desired type of dataset. - -**The dataset loaders.** They can be used to load small standard datasets, -described in the :ref:`toy_datasets` section. + +**The dataset loaders.** They can be used to load small standard datasets, +described in the :ref:`toy_datasets` section. **The dataset fetchers.** They can be used to download and load larger datasets, described in the :ref:`real_world_datasets` section. -Both loaders and fetchers functions return a dictionary-like object holding -at least two items: an array of shape ``n_samples`` * ``n_features`` with -key ``data`` (except for 20newsgroups) and a numpy array of +Both loaders and fetchers functions return a :class:`sklearn.utils.Bunch` +object holding at least two items: +an array of shape ``n_samples`` * ``n_features`` with +key ``data`` (except for 20newsgroups) and a numpy array of length ``n_samples``, containing the target values, with key ``target``. +The Bunch object is a dictionary that exposes its keys are attributes. +For more information about Bunch object, see :class:`sklearn.utils.Bunch`: + It's also possible for almost all of these function to constrain the output -to be a tuple containing only the data and the target, by setting the +to be a tuple containing only the data and the target, by setting the ``return_X_y`` parameter to ``True``. -The datasets also contain a full description in their ``DESCR`` attribute and -some contain ``feature_names`` and ``target_names``. See the dataset -descriptions below for details. +The datasets also contain a full description in their ``DESCR`` attribute and +some contain ``feature_names`` and ``target_names``. See the dataset +descriptions below for details. -**The dataset generation functions.** They can be used to generate controlled +**The dataset generation functions.** They can be used to generate controlled synthetic datasets, described in the :ref:`sample_generators` section. These functions return a tuple ``(X, y)`` consisting of a ``n_samples`` * ``n_features`` numpy array ``X`` and an array of length ``n_samples`` containing the targets ``y``. -In addition, there are also miscellaneous tools to load datasets of other +In addition, there are also miscellaneous tools to load datasets of other formats or from other locations, described in the :ref:`loading_other_datasets` -section. +section. .. _toy_datasets: Toy datasets ============ -scikit-learn comes with a few small standard datasets that do not require to -download any file from some external website. +scikit-learn comes with a few small standard datasets that do not require to +download any file from some external website. They can be loaded using the following functions: @@ -484,17 +488,17 @@ Loading from external datasets scikit-learn works on any numeric data stored as numpy arrays or scipy sparse matrices. Other types that are convertible to numeric arrays such as pandas DataFrame are also acceptable. - -Here are some recommended ways to load standard columnar data into a -format usable by scikit-learn: -* `pandas.io `_ +Here are some recommended ways to load standard columnar data into a +format usable by scikit-learn: + +* `pandas.io `_ provides tools to read data from common formats including CSV, Excel, JSON and SQL. DataFrames may also be constructed from lists of tuples or dicts. Pandas handles heterogeneous data smoothly and provides tools for manipulation and conversion into a numeric array suitable for scikit-learn. -* `scipy.io `_ - specializes in binary formats often used in scientific computing +* `scipy.io `_ + specializes in binary formats often used in scientific computing context such as .mat and .arff * `numpy/routines.io `_ for standard loading of columnar data into numpy arrays @@ -508,18 +512,18 @@ For some miscellaneous data such as images, videos, and audio, you may wish to refer to: * `skimage.io `_ or - `Imageio `_ + `Imageio `_ for loading images and videos into numpy arrays -* `scipy.io.wavfile.read - `_ +* `scipy.io.wavfile.read + `_ for reading WAV files into a numpy array -Categorical (or nominal) features stored as strings (common in pandas DataFrames) +Categorical (or nominal) features stored as strings (common in pandas DataFrames) will need converting to numerical features using :class:`sklearn.preprocessing.OneHotEncoder` or :class:`sklearn.preprocessing.OrdinalEncoder` or similar. See :ref:`preprocessing`. -Note: if you manage your own numerical data it is recommended to use an +Note: if you manage your own numerical data it is recommended to use an optimized file format such as HDF5 to reduce data load times. Various libraries -such as H5Py, PyTables and pandas provides a Python interface for reading and +such as H5Py, PyTables and pandas provides a Python interface for reading and writing data in that format. diff --git a/doc/developers/advanced_installation.rst b/doc/developers/advanced_installation.rst index f5b5f18521e34..35da69fe63563 100644 --- a/doc/developers/advanced_installation.rst +++ b/doc/developers/advanced_installation.rst @@ -26,7 +26,7 @@ Installing a nightly build is the quickest way to: :: - pip install --pre -f https://sklearn-nightly.scdn8.secure.raxcdn.com scikit-learn + pip install --pre --extra-index https://pypi.anaconda.org/scipy-wheels-nightly/simple scikit-learn .. _install_bleeding_edge: @@ -43,13 +43,13 @@ feature, code or documentation improvement). `scikit-learn repository `_ on Github.:: - git clone git://github.com/scikit-learn/scikit-learn.git + git clone git://github.com/scikit-learn/scikit-learn.git # add --depth 1 if your connection is slow cd scikit-learn If you plan on submitting a pull-request, you should clone from your fork instead. -#. Install a compiler with OpenMP_ support for your platform. See intructions +#. Install a compiler with OpenMP_ support for your platform. See instructions for :ref:`compiler_windows`, :ref:`compiler_macos`, :ref:`compiler_linux` and :ref:`compiler_freebsd`. @@ -59,7 +59,7 @@ feature, code or documentation improvement). #. Install Cython_ and build the project with pip in :ref:`editable_mode`:: pip install cython - pip install --verbose --editable . + pip install --verbose --no-build-isolation --editable . #. Check that the installed scikit-learn has a version number ending with `.dev0`:: @@ -71,8 +71,11 @@ feature, code or documentation improvement). .. note:: - You will have to re-run the ``pip install --editable .`` command every time - the source code of a Cython file is updated (ending in `.pyx` or `.pxd`). + You will have to run the ``pip install --no-build-isolation --editable .`` + command every time the source code of a Cython file is updated + (ending in `.pyx` or `.pxd`). Use the ``--no-build-isolation`` flag to + avoid compiling the whole project each time, only the files you have + modified. Dependencies ------------ @@ -83,9 +86,9 @@ Runtime dependencies Scikit-learn requires the following dependencies both at build time and at runtime: -- Python (>= 3.5), -- NumPy (>= 1.11), -- SciPy (>= 0.17), +- Python (>= 3.6), +- NumPy (>= 1.13.3), +- SciPy (>= 0.19), - Joblib (>= 0.11). Those dependencies are **automatically installed by pip** if they were missing @@ -152,9 +155,9 @@ Editable mode If you run the development version, it is cumbersome to reinstall the package each time you update the sources. Therefore it is recommended that you install -in with the ``pip install --editable .`` command, which allows you to edit the -code in-place. This builds the extension in place and creates a link to the -development directory (see `the pip docs +in with the ``pip install --no-build-isolation --editable .`` command, which +allows you to edit the code in-place. This builds the extension in place and +creates a link to the development directory (see `the pip docs `_). This is fundamentally similar to using the command ``python setup.py develop`` @@ -207,7 +210,7 @@ environment variables in the current command prompt. Finally, build scikit-learn from this command prompt:: - pip install --verbose --editable . + pip install --verbose --no-build-isolation --editable . .. _compiler_macos: @@ -225,9 +228,9 @@ to enable OpenMP support: macOS compilers from conda-forge ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you use the conda package manager, you can install the ``compilers`` -meta-package from the conda-forge channel, which provides OpenMP-enabled C/C++ -compilers based on the llvm toolchain. +If you use the conda package manager (version >= 4.7), you can install the +``compilers`` meta-package from the conda-forge channel, which provides +OpenMP-enabled C/C++ compilers based on the llvm toolchain. First install the macOS command line tools:: @@ -237,10 +240,10 @@ It is recommended to use a dedicated `conda environment`_ to build scikit-learn from source:: conda create -n sklearn-dev python numpy scipy cython joblib pytest \ - conda-forge::compilers conda-forge::llvm-openmp + "conda-forge::compilers>=1.0.4" conda-forge::llvm-openmp conda activate sklearn-dev make clean - pip install --verbose --editable . + pip install --verbose --no-build-isolation --editable . .. note:: @@ -252,7 +255,9 @@ scikit-learn from source:: You can check that the custom compilers are properly installed from conda forge using the following command:: - conda list compilers llvm-openmp + conda list + +which should include ``compilers`` and ``llvm-openmp``. The compilers meta-package will automatically set custom environment variables:: @@ -264,7 +269,8 @@ variables:: echo $LDFLAGS They point to files and folders from your ``sklearn-dev`` conda environment -(in particular in the bin/, include/ and lib/ subfolders). +(in particular in the bin/, include/ and lib/ subfolders). For instance +``-L/path/to/conda/envs/sklearn-dev/lib`` should appear in ``LDFLAGS``. In the log, you should see the compiled extension being built with the clang and clang++ compilers installed by conda with the ``-fopenmp`` command line @@ -299,7 +305,7 @@ Finally, build scikit-learn in verbose mode (to check for the presence of the ``-fopenmp`` flag in the compiler commands):: make clean - pip install --verbose --editable . + pip install --verbose --no-build-isolation --editable . .. _compiler_linux: @@ -338,7 +344,7 @@ architecture (e.g. ARM), you can install the system versions:: On Red Hat and clones (e.g. CentOS), install the dependencies using:: - sudo yum -y install gcc gcc-c++ python-devel numpy scipy + sudo yum -y install gcc gcc-c++ python3-devel numpy scipy Linux compilers from conda-forge ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -348,7 +354,7 @@ in the user folder using conda:: conda create -n sklearn-dev numpy scipy joblib cython conda-forge::compilers conda activate sklearn-dev - pip install --verbose --editable . + pip install --verbose --no-build-isolation --editable . .. _compiler_freebsd: @@ -371,9 +377,9 @@ can set the environment variables to these locations:: Finally, build the package using the standard command:: - pip install --verbose --editable . + pip install --verbose --no-build-isolation --editable . -For the upcomming FreeBSD 12.1 and 11.3 versions, OpenMP will be included in +For the upcoming FreeBSD 12.1 and 11.3 versions, OpenMP will be included in the base system and these steps will not be necessary. .. _OpenMP: https://en.wikipedia.org/wiki/OpenMP diff --git a/doc/developers/contributing.rst b/doc/developers/contributing.rst index 863ecfb7741b3..3b2f7317ee41b 100644 --- a/doc/developers/contributing.rst +++ b/doc/developers/contributing.rst @@ -181,7 +181,12 @@ Contributing code If in doubt about duplicated work, or if you want to work on a non-trivial feature, it's recommended to first open an issue in the `issue tracker `_ - to get some feedbacks from core developers. + to get some feedbacks from core developers. + + One easy way to find an issue to work on is by applying the "help wanted" + label in your search. This lists all the issues that have been unclaimed + so far. In order to claim an issue for yourself, please comment exactly + ``take`` on it for the CI to automatically assign the issue to you. How to contribute ----------------- @@ -205,7 +210,7 @@ how to set up your git repository: 3. Clone your fork of the scikit-learn repo from your GitHub account to your local disk:: - $ git clone git@github.com:YourLogin/scikit-learn.git + $ git clone git@github.com:YourLogin/scikit-learn.git # add --depth 1 if your connection is slow $ cd scikit-learn 4. Install the development dependencies:: @@ -214,7 +219,7 @@ how to set up your git repository: 5. Install scikit-learn in editable mode:: - $ pip install --editable . + $ pip install --no-build-isolation --editable . for more details about advanced installation, see the :ref:`install_bleeding_edge` section. @@ -251,7 +256,7 @@ modifying code and submitting a PR: to record your changes in Git, then push the changes to your GitHub account with:: - $ git push -u origin my-feature + $ git push -u origin my_feature 10. Follow `these `_ @@ -261,8 +266,13 @@ modifying code and submitting a PR: .. note:: - If you are modifying a Cython module, you have to re-run step 5 after modifications - and before testing them. + If you are modifying a Cython module, you have to re-compile after + modifications and before testing them:: + + pip install --no-build-isolation -e . + + Use the ``--no-build-isolation`` flag to avoid compiling the whole project + each time, only the files you have modified. It is often helpful to keep your local feature branch synchronized with the latest changes of the main scikit-learn repository:: @@ -377,7 +387,7 @@ complies with the following rules before marking a PR as ``[MRG]``. The methods available in scikit-learn. 10. New features often need to be illustrated with narrative documentation in - the user guide, with small code snipets. If relevant, please also add + the user guide, with small code snippets. If relevant, please also add references in the literature, with PDF links when possible. 11. The user guide should also include expected time and space complexity @@ -437,6 +447,7 @@ message, the following actions are taken. ---------------------- ------------------- [scipy-dev] Add a Travis build with our dependencies (numpy, scipy, etc ...) development builds [ci skip] CI is skipped completely + [lint skip] Azure pipeline skips linting [doc skip] Docs are not built [doc quick] Docs built, but excludes example gallery plots [doc build] Docs built including example gallery plots @@ -676,22 +687,8 @@ Generated documentation on CircleCI ----------------------------------- When you change the documentation in a pull request, CircleCI automatically -builds it. To view the documentation generated by CircleCI: - -* navigate to the bottom of your pull request page to see the CI - statuses. You may need to click on "Show all checks" to see all the CI - statuses. -* click on the CircleCI status with "doc" in the title. -* add ``#artifacts`` at the end of the URL. Note: you need to wait for the - CircleCI build to finish before being able to look at the artifacts. -* once the artifacts are visible, navigate to ``doc/_changed.html`` to see a - list of documentation pages that are likely to be affected by your pull - request. Navigate to ``doc/index.html`` to see the full generated html - documentation. - -If you often need to look at the documentation generated by CircleCI, e.g. when -reviewing pull requests, you may find :ref:`this tip -` very handy. +builds it. To view the documentation generated by CircleCI, simply go at the +bottom of your PR page and look for the "ci/circleci: doc artifact" link. .. _testing_coverage: diff --git a/doc/developers/develop.rst b/doc/developers/develop.rst index ead6286d98083..96aa942fb9238 100644 --- a/doc/developers/develop.rst +++ b/doc/developers/develop.rst @@ -74,7 +74,7 @@ multiple interfaces): Estimators ---------- -The API has one predominant object: the estimator. A estimator is an +The API has one predominant object: the estimator. An estimator is an object that fits a model based on some training data and is capable of inferring some properties on new data. It can be, for instance, a classifier or a regressor. All estimators implement the fit method:: @@ -220,12 +220,23 @@ an integer called ``n_iter``. Pairwise Attributes ^^^^^^^^^^^^^^^^^^^ -An estimator that accept ``X`` of shape ``(n_samples, n_samples)`` and defines +An estimator that accepts ``X`` of shape ``(n_samples, n_samples)`` and defines a :term:`_pairwise` property equal to ``True`` allows for cross-validation of the dataset, e.g. when ``X`` is a precomputed kernel matrix. Specifically, the :term:`_pairwise` property is used by ``utils.metaestimators._safe_split`` to slice rows and columns. +Universal attributes +^^^^^^^^^^^^^^^^^^^^ + +Estimators that expect tabular input should set a `n_features_in_` +attribute at `fit` time to indicate the number of features that the estimator +expects for subsequent calls to `predict` or `transform`. +See +`SLEP010 +`_ +for details. + .. _rolling_your_own_estimator: Rolling your own estimator @@ -468,50 +479,44 @@ Estimator Tags The estimator tags are experimental and the API is subject to change. -Scikit-learn introduced estimator tags in version 0.21. These are annotations +Scikit-learn introduced estimator tags in version 0.21. These are annotations of estimators that allow programmatic inspection of their capabilities, such as -sparse matrix support, supported output types and supported methods. The -estimator tags are a dictionary returned by the method ``_get_tags()``. These -tags are used by the common tests and the :func:`sklearn.utils.estimator_checks.check_estimator` function to -decide what tests to run and what input data is appropriate. Tags can depend on -estimator parameters or even system architecture and can in general only be -determined at runtime. - -The default value of all tags except for ``X_types`` and ``requires_fit`` is -``False``. These are defined in the ``BaseEstimator`` class. +sparse matrix support, supported output types and supported methods. The +estimator tags are a dictionary returned by the method ``_get_tags()``. These +tags are used by the common tests and the +:func:`sklearn.utils.estimator_checks.check_estimator` function to decide what +tests to run and what input data is appropriate. Tags can depend on estimator +parameters or even system architecture and can in general only be determined at +runtime. The default values for the estimator tags are defined in the +``BaseEstimator`` class. The current set of estimator tags are: -non_deterministic - whether the estimator is not deterministic given a fixed ``random_state`` - -requires_positive_X - whether the estimator requires positive X. - -requires_positive_y - whether the estimator requires a positive y (only applicable for regression). - -no_validation - whether the estimator skips input-validation. This is only meant for stateless and dummy transformers! +allow_nan (default=False) + whether the estimator supports data with missing values encoded as np.NaN -multioutput - unused for now - whether a regressor supports multi-target outputs or a classifier supports multi-class multi-output. +binary_only (default=False) + whether estimator supports binary classification but lacks multi-class + classification support. -multilabel +multilabel (default=False) whether the estimator supports multilabel output -stateless - whether the estimator needs access to data for fitting. Even though - an estimator is stateless, it might still need a call to ``fit`` for initialization. +multioutput (default=False) + whether a regressor supports multi-target outputs or a classifier supports + multi-class multi-output. -requires_fit - whether the estimator requires to be fitted before calling one of - `transform`, `predict`, `predict_proba`, or `decision_function`. +multioutput_only (default=False) + whether estimator supports only multi-output classification or regression. -allow_nan - whether the estimator supports data with missing values encoded as np.NaN +no_validation (default=False) + whether the estimator skips input-validation. This is only meant for + stateless and dummy transformers! -poor_score +non_deterministic (default=False) + whether the estimator is not deterministic given a fixed ``random_state`` + +poor_score (default=False) whether the estimator fails to provide a "reasonable" test-set score, which currently for regression is an R2 of 0.5 on a subset of the boston housing dataset, and for classification an accuracy of 0.83 on @@ -519,24 +524,39 @@ poor_score are based on current estimators in sklearn and might be replaced by something more systematic. -multioutput_only - whether estimator supports only multi-output classification or regression. +requires_fit (default=True) + whether the estimator requires to be fitted before calling one of + `transform`, `predict`, `predict_proba`, or `decision_function`. -binary_only - whether estimator supports binary classification but lacks multi-class - classification support. +requires_positive_X (default=False) + whether the estimator requires positive X. + +requires_positive_y (default=False) + whether the estimator requires a positive y (only applicable for regression). -_skip_test - whether to skip common tests entirely. Don't use this unless you have a *very good* reason. - -X_types - Supported input types for X as list of strings. Tests are currently only run if '2darray' is contained - in the list, signifying that the estimator takes continuous 2d numpy arrays as input. The default - value is ['2darray']. Other possible types are ``'string'``, ``'sparse'``, - ``'categorical'``, ``dict``, ``'1dlabels'`` and ``'2dlabels'``. - The goal is that in the future the supported input type will determine the - data used during testing, in particular for ``'string'``, ``'sparse'`` and - ``'categorical'`` data. For now, the test for sparse data do not make use +_skip_test (default=False) + whether to skip common tests entirely. Don't use this unless you have a + *very good* reason. + +_xfail_test (default=False) + dictionary ``{check_name : reason}`` of common checks to mark as a + known failure, with the associated reason. Don't use this unless you have a + *very good* reason. + +stateless (default=False) + whether the estimator needs access to data for fitting. Even though an + estimator is stateless, it might still need a call to ``fit`` for + initialization. + +X_types (default=['2darray']) + Supported input types for X as list of strings. Tests are currently only + run if '2darray' is contained in the list, signifying that the estimator + takes continuous 2d numpy arrays as input. The default value is + ['2darray']. Other possible types are ``'string'``, ``'sparse'``, + ``'categorical'``, ``dict``, ``'1dlabels'`` and ``'2dlabels'``. The goal is + that in the future the supported input type will determine the data used + during testing, in particular for ``'string'``, ``'sparse'`` and + ``'categorical'`` data. For now, the test for sparse data do not make use of the ``'sparse'`` tag. @@ -601,7 +621,7 @@ In addition, we add the following guidelines: find bugs in scikit-learn. * Use the `numpy docstring standard - `_ in all your docstrings. + `_ in all your docstrings. A good example of code that we like can be found `here diff --git a/doc/developers/maintainer.rst b/doc/developers/maintainer.rst index e91f01999b12e..2a42bee301554 100644 --- a/doc/developers/maintainer.rst +++ b/doc/developers/maintainer.rst @@ -1,14 +1,27 @@ Maintainer / core-developer information ======================================== + +Releasing +--------- + +This section is about preparing a major release, incrementing the minor +version, or a bug fix release incrementing the patch version. Our convention is +that we release one or more release candidates (0.RRrcN) before releasing the +final distributions. We follow the `PEP101 +`_ to indicate release candidates, +post, and minor releases. + Before a release ----------------- +................ 1. Update authors table:: $ cd build_tools; make authors; cd .. - and commit. + and commit. This is only needed if the authors have changed since the last + release. This step is sometimes done independent of the release. This + updates the maintainer list and is not the contributor list for the release. 2. Confirm any blockers tagged for the milestone are resolved, and that other issues tagged for the milestone can be postponed. @@ -17,61 +30,98 @@ Before a release change log is reasonably well curated. Some tools for these tasks include: - ``maint_tools/sort_whats_new.py`` can put what's new entries into - sections. + sections. It's not perfect, and requires manual checking of the changes. + If the whats new list is well curated, it may not be necessary. - The ``maint_tools/whats_missing.sh`` script may be used to identify pull requests that were merged but likely missing from What's New. -Preparing a bug-fix-release -........................... +4. Make sure the deprecations, FIXME and TODOs tagged for the release have + been taken care of. + +**Permissions** + +The release manager requires a set of permissions on top of the usual +permissions given to maintainers, which includes: + +- *maintainer* role on ``scikit-learn`` projects on ``pypi.org`` and + ``test.pypi.org``, separately. +- become a member of the *scikit-learn* team on conda-forge by editing the + ``recipe/meta.yaml`` file on + ``https://github.com/conda-forge/scikit-learn-feedstock`` +- *maintainer* on ``https://github.com/MacPython/scikit-learn-wheels`` + + +.. _preparing_a_release_pr: + +Preparing a release PR +...................... -Since any commits to a released branch (e.g. 0.999.X) will automatically update -the web site documentation, it is best to develop a bug-fix release with a pull -request in which 0.999.X is the base. It also allows you to keep track of any -tasks towards release with a TO DO list. +Releasing the first RC of e.g. version `0.99` involves creating the release +branch `0.99.X` directly on the main repo, where `X` really is the letter X, +**not a placeholder**. This is considered the *feature freeze*. The +development for the major and minor releases of 0.99 should +**also** happen under `0.99.X`. Each release (rc, major, or minor) is a tag +under that branch. -Most development of the bug fix release, and its documentation, should -happen in master to avoid asynchrony. To select commits from master for use in -the bug fix (version 0.999.3), you can use:: +In terms of including changes, the first RC ideally counts as a *feature +freeze*. Each coming release candidate and the final release afterwards will +include minor documentation changes and bug fixes. Any major enhancement or +feature should be excluded. +The minor releases should include bug fixes and some relevant documentation +changes only. Any PR resulting in a behavior change which is not a bug fix +should be excluded. + +First, create a branch, **on your own fork** (to release e.g. `0.999.3`):: + + $ # assuming master and upstream/master are the same $ git checkout -b release-0.999.3 master - $ git rebase -i 0.999.X -Then pick the commits for release and resolve any issues, and create a pull -request with 0.999.X as base. Add a commit updating ``sklearn.__version__``. -Additional commits can be cherry-picked into the ``release-0.999.3`` branch -while preparing the release. +Then, create a PR **to the** `scikit-learn/0.999.X` **branch** (not to +master!) with all the desired changes:: + + $ git rebase -i upstream/0.999.2 + +Do not forget to add a commit updating sklearn.__version__. + +It's nice to have a copy of the ``git rebase -i`` log in the PR to help others +understand what's included. Making a release ----------------- +................ + +0. Create the release branch on the main repo, if it does not exist. This is + done only once, as the major and minor releases happen on the same branch:: + + $ git checkout -b 0.99.X + + Again, `X` is literal here, and `99` is replaced by the release number. + The branches are called ``0.19.X``, ``0.20.X``, etc. -1. Update docs: +1. Update docs. Note that this is for the final release, not necessarily for + the RC releases. These changes should be made in master and cherry-picked + into the release branch, only before the final release. - Edit the doc/whats_new.rst file to add release title and commit statistics. You can retrieve commit statistics with:: $ git shortlog -s 0.99.33.. | cut -f2- | sort --ignore-case | tr '\n' ';' | sed 's/;/, /g;s/, $//' - - Update the release date in whats_new.rst + - Update the release date in ``whats_new.rst`` - - Edit the doc/index.rst to change the 'News' entry of the front page. - - - Note that these changes should be made in master and cherry-picked into - the release branch. + - Edit the doc/templates/index.html to change the 'News' entry of the front + page. 2. On the branch for releasing, update the version number in - sklearn/__init__.py, the ``__version__`` variable by removing ``dev*`` only - when ready to release. - On master, increment the verson in the same place (when branching for - release). - -3. Create the tag and push it:: - - $ git tag -a 0.999 + `sklearn/__init__.py`, the ``__version__`` variable by removing ``dev*`` + only when ready to release. On master, increment the version in the same + place (when branching for release). This means while we're in the release + candidate period, the latest stable is two versions behind the master + branch, instead of one. - $ git push git@github.com:scikit-learn/scikit-learn.git --tags - -4. Create the source tarball: +3. At this point all relevant PRs should have been merged into the `0.99.X` + branch. Create the source tarball: - Wipe clean your repo:: @@ -81,10 +131,32 @@ Making a release $ python setup.py sdist + - You can also test a binary dist build using:: + + $ python setup.py bdist_wheel + + - You can test if PyPi is going to accept the package using:: + + $ twine check dist/* + + You can run ``twine check`` after step 5 (fetching artifacts) as well. + The result should be in the `dist/` folder. We will upload it later with the wheels. Check that you can install it in a new virtualenv and that the tests pass. +4. Proceed with caution. Ideally, tags should be created when you're almost + certain that the release is ready, since adding a tag to the main repo can + trigger certain automated processes. You can test upload the ``sdist`` to + ``test.pypi.org``, and test the next step by setting ``BUILD_COMMIT`` to the + branch name (``0.99.X`` for instance) in a PR to the wheel building repo. + Once all works, you can proceed with tagging. Create the tag and push it (if + it's an RC, it can be ``0.xxrc1`` for instance):: + + $ git tag -a 0.99 # in the 0.99.X branch + + $ git push git@github.com:scikit-learn/scikit-learn.git 0.99 + 5. Update the dependency versions and set ``BUILD_COMMIT`` variable to the release tag at: @@ -94,16 +166,20 @@ Making a release packages and upload them to PyPI by running the following commands in the scikit-learn source folder (checked out at the release tag):: - $ rm -r dist + $ rm -r dist # only if there's anything other than the sdist tar.gz there $ pip install -U wheelhouse_uploader twine $ python setup.py fetch_artifacts 6. Check the content of the `dist/` folder: it should contain all the wheels - along with the source tarball ("scikit-learn-XXX.tar.gz"). + along with the source tarball ("scikit-learn-RRR.tar.gz"). Make sure that you do not have developer versions or older versions of the scikit-learn package in that folder. + Before uploading to pypi, you can test upload to test.pypi.org:: + + $ twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/* + Upload everything at once to https://pypi.org:: $ twine upload dist/* @@ -119,21 +195,43 @@ Making a release $ git checkout master $ rm stable $ ln -s 0.999 stable - $ sed -i "s/latestStable = '.*/latestStable = '0.999';" versionwarning.js - $ git commit -m "Update stable to point to 0.999" stable + $ sed -i "s/latestStable = '.*/latestStable = '0.999';/" versionwarning.js + $ git add stable/ versionwarning.js + $ git commit -m "Update stable to point to 0.999" $ git push origin master The following GitHub checklist might be helpful in a release PR:: * [ ] update news and what's new date in master and release branch * [ ] create tag - * [ ] update dependencies and release tag at https://github.com/MacPython/scikit-learn-wheels + * [ ] update dependencies and release tag at + https://github.com/MacPython/scikit-learn-wheels * [ ] twine the wheels to PyPI when that's green * [ ] https://github.com/scikit-learn/scikit-learn/releases draft - * [ ] confirm bot detected at https://github.com/conda-forge/scikit-learn-feedstock and wait for merge + * [ ] confirm bot detected at + https://github.com/conda-forge/scikit-learn-feedstock and wait for merge * [ ] https://github.com/scikit-learn/scikit-learn/releases publish - * [ ] announce on mailing list - * [ ] (regenerate Dash docs: https://github.com/Kapeli/Dash-User-Contributions/tree/master/docsets/Scikit) + * [ ] fix the binder release version in ``.binder/requirement.txt`` (see + #15847) + * [ ] announce on mailing list and on twitter + +Merging Pull Requests +--------------------- + +Individual commits are squashed when a Pull Request (PR) is merged on Github. +Before merging, + +- the resulting commit title can be edited if necessary. Note + that this will rename the PR title by default. +- the detailed description, containing the titles of all the commits, can + be edited or deleted. +- for PRs with multiple code contributors care must be taken to keep + the `Co-authored-by: name ` tags in the detailed + description. This will mark the PR as having `multiple co-authors + `_. + Whether code contributions are significanly enough to merit co-authorship is + left to the maintainer's discretion, same as for the "what's new" entry. + The scikit-learn.org web site ----------------------------- diff --git a/doc/developers/tips.rst b/doc/developers/tips.rst index 4c313701b4aa6..b26d68ecfbe02 100644 --- a/doc/developers/tips.rst +++ b/doc/developers/tips.rst @@ -17,21 +17,6 @@ as GitHub gists; to install them, click on the "Raw" button on the gist page. .. _TamperMonkey: https://tampermonkey.net/ .. _GreaseMonkey: https://www.greasespot.net/ - -.. _viewing_rendered_html_documentation: - -Viewing the rendered HTML documentation for a pull request ----------------------------------------------------------- - -We use CircleCI to build the HTML documentation for every pull request. To -access that documentation, instructions are provided in the :ref:`documentation -section of the contributor guide `. To save you a few -clicks, we provide a `userscript -`__ -that adds a button to every PR. After installing the userscript, navigate to -any GitHub PR; a new button labeled "See CircleCI doc for this PR" should -appear in the top-right area. - Folding and unfolding outdated diffs on pull requests ----------------------------------------------------- @@ -101,6 +86,8 @@ Other `pytest` options that may become useful include: - ``-s`` so that pytest does not capture the output of ``print()`` statements - ``--tb=short`` or ``--tb=line`` to control the length of the logs + - ``--runxfail`` also run tests marked as a known failure (XFAIL) and report + errors. Since our continuous integration tests will error if ``FutureWarning`` isn't properly caught, @@ -181,10 +168,10 @@ Issue/Comment: Linking to comments Please use links to comments, which make it a lot easier to see what you are referring to, rather than just linking to the issue. See [this](https://stackoverflow.com/questions/25163598/how-do-i-reference-a-specific-issue-comment-on-github) for more details. -PR-NEW: Better description +PR-NEW: Better description and title :: - Thanks for the pull request! Please make the title of the PR descriptive so that we can easily recall the issue it is resolving. You should state what issue (or PR) it fixes/resolves in the description (see [here](http://scikit-learn.org/dev/developers/contributing.html#contributing-pull-requests)). + Thanks for the pull request! Please make the title of the PR more descriptive. The title will become the commit message when this is merged. You should state what issue (or PR) it fixes/resolves in the description using the syntax described [here](http://scikit-learn.org/dev/developers/contributing.html#contributing-pull-requests). PR-NEW: Fix # :: diff --git a/doc/faq.rst b/doc/faq.rst index 1b216dd3d1dab..883ac290a3f16 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -74,14 +74,12 @@ See :ref:`model_persistence`. How can I create a bunch object? ------------------------------------------------ -Don't make a bunch object! They are not part of the scikit-learn API. Bunch -objects are just a way to package some numpy arrays. As a scikit-learn user you -only ever need numpy arrays to feed your model with data. +Bunch objects are sometimes used as an output for functions and methods. They +extend dictionaries by enabling values to be accessed by key, +`bunch["value_key"]`, or by an attribute, `bunch.value_key`. -For instance to train a classifier, all you need is a 2D array ``X`` for the -input variables and a 1D array ``y`` for the target variables. The array ``X`` -holds the features as columns and samples as rows . The array ``y`` contains -integer values to encode the class membership of each sample in ``X``. +They should not be used as an input; therefore you almost never need to create +a ``Bunch`` object, unless you are extending the scikit-learn's API. How can I load my own datasets into a format usable by scikit-learn? -------------------------------------------------------------------- @@ -328,6 +326,14 @@ scikit-learn seeks to achieve. You can find more information about addition of gpu support at `Will you add GPU support?`_. +Note that scikit-learn currently implements a simple multilayer perceptron +in `sklearn.neural_network`. We will only accept bug fixes for this module. +If you want to implement more complex deep learning models, please turn to +popular deep learning frameworks such as +`tensorflow `_, +`keras `_ +and `pytorch `_. + Why is my pull request not getting any attention? ------------------------------------------------- @@ -385,3 +391,23 @@ efficient to process for most operations. Extensive work would also be needed to support Pandas categorical types. Restricting input to homogeneous types therefore reduces maintenance cost and encourages usage of efficient data structures. + +Do you plan to implement transform for target y in a pipeline? +---------------------------------------------------------------------------- +Currently transform only works for features X in a pipeline. +There's a long-standing discussion about +not being able to transform y in a pipeline. +Follow on github issue +`#4143 `_. +Meanwhile check out +:class:`sklearn.compose.TransformedTargetRegressor`, +`pipegraph `_, +`imbalanced-learn `_. +Note that Scikit-learn solved for the case where y +has an invertible transformation applied before training +and inverted after prediction. Scikit-learn intends to solve for +use cases where y should be transformed at training time +and not at test time, for resampling and similar uses, +like at imbalanced learn. +In general, these use cases can be solved +with a custom meta estimator rather than a Pipeline diff --git a/doc/getting_started.rst b/doc/getting_started.rst index ba18b92e40983..79e7ac5b52bb9 100644 --- a/doc/getting_started.rst +++ b/doc/getting_started.rst @@ -205,7 +205,7 @@ the best set of parameters. Read more in the :ref:`User Guide training and testing data. Indeed, since you pre-processed the data using the whole dataset, some information about the test sets are available to the train sets. This will lead to over-estimating the - generalization power of the estimator (you can read more in this `kaggle + generalization power of the estimator (you can read more in this `Kaggle post `_). Using a pipeline for cross-validation and searching will largely keep diff --git a/doc/glossary.rst b/doc/glossary.rst index 3a01b76a45781..70dd230c7ecd3 100644 --- a/doc/glossary.rst +++ b/doc/glossary.rst @@ -374,6 +374,8 @@ General Concepts the data needs to be indexed on both axes, while other data is indexed only on the first axis. + For more detailed info, see :ref:`estimator_tags`. + feature features feature vector @@ -1161,7 +1163,7 @@ Methods TODO: `This gist `_ - higlights the use of the different formats for multilabel. + highlights the use of the different formats for multilabel. multioutput classification A list of 2d arrays, corresponding to each multiclass decision function. diff --git a/doc/install.rst b/doc/install.rst index d45e9f3367ff1..6a2b83605c1a6 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -126,20 +126,22 @@ If you have not installed NumPy or SciPy yet, you can also install these using conda or pip. When using pip, please ensure that *binary wheels* are used, and NumPy and SciPy are not recompiled from source, which can happen when using particular configurations of operating system and hardware (such as Linux on -a Raspberry Pi). +a Raspberry Pi). If you must install scikit-learn and its dependencies with pip, you can install it as ``scikit-learn[alldeps]``. Scikit-learn plotting capabilities (i.e., functions start with "plot\_" -and classes end with "Display") require Matplotlib (>= 1.5.1). For running the -examples Matplotlib >= 1.5.1 is required. A few examples require -scikit-image >= 0.12.3, a few examples require pandas >= 0.18.0. +and classes end with "Display") require Matplotlib (>= 2.1.1). For running the +examples Matplotlib >= 2.1.1 is required. A few examples require +scikit-image >= 0.13, a few examples require pandas >= 0.18.0. .. warning:: Scikit-learn 0.20 was the last version to support Python 2.7 and Python 3.4. - Scikit-learn now requires Python 3.5 or newer. + Scikit-learn 0.21 supported Python 3.5-3.7. + Scikit-learn 0.22 supported Python 3.5-3.8. + Scikit-learn now requires Python 3.6 or newer. .. note:: @@ -230,8 +232,63 @@ library for Windows, Mac OSX and Linux. Anaconda offers scikit-learn as part of its free distribution. +Intel conda channel +------------------- + +Intel maintains a dedicated conda channel that ships scikit-learn:: + + $ conda install -c intel scikit-learn + +This version of scikit-learn comes with alternative solvers for some common +estimators. Those solvers come from the DAAL C++ library and are optimized for +multi-core Intel CPUs. + +Note that those solvers are not enabled by default, please refer to the +`daal4py `_ documentation +for more details. + +Compatibility with the standard scikit-learn solvers is checked by running the +full scikit-learn test suite via automated continuous integration as reported +on https://github.com/IntelPython/daal4py. + + WinPython for Windows ----------------------- The `WinPython `_ project distributes scikit-learn as an additional plugin. + + +Troubleshooting +=============== + +.. _windows_longpath: + +Error caused by file path length limit on Windows +------------------------------------------------- + +It can happen that pip fails to install packages when reaching the default path +size limit of Windows if Python is installed in a nested location such as the +`AppData` folder structure under the user home directory, for instance:: + + C:\Users\username>C:\Users\username\AppData\Local\Microsoft\WindowsApps\python.exe -m pip install scikit-learn + Collecting scikit-learn + ... + Installing collected packages: scikit-learn + ERROR: Could not install packages due to an EnvironmentError: [Errno 2] No such file or directory: 'C:\\Users\\username\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.7_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python37\\site-packages\\sklearn\\datasets\\tests\\data\\openml\\292\\api-v1-json-data-list-data_name-australian-limit-2-data_version-1-status-deactivated.json.gz' + +In this case it is possible to lift that limit in the Windows registry by +using the ``regedit`` tool: + +#. Type "regedit" in the Windows start menu to launch ``regedit``. + +#. Go to the + ``Computer\HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem`` + key. + +#. Edit the value of the ``LongPathsEnabled`` property of that key and set + it to 1. + +#. Reinstall scikit-learn (ignoring the previous broken installation):: + + pip install --exists-action=i scikit-learn diff --git a/doc/modules/calibration.rst b/doc/modules/calibration.rst index 6fe30c93ff142..19df08ea3b1fe 100644 --- a/doc/modules/calibration.rst +++ b/doc/modules/calibration.rst @@ -19,8 +19,16 @@ Well calibrated classifiers are probabilistic classifiers for which the output of the predict_proba method can be directly interpreted as a confidence level. For instance, a well calibrated (binary) classifier should classify the samples such that among the samples to which it gave a predict_proba value close to 0.8, -approximately 80% actually belong to the positive class. The following plot compares -how well the probabilistic predictions of different classifiers are calibrated: +approximately 80% actually belong to the positive class. + +Calibration curves +------------------ + +The following plot compares how well the probabilistic predictions of +different classifiers are calibrated, using :func:`calibration_curve`. +The x axis represents the average predicted probability in each bin. The +y axis is the *fraction of positives*, i.e. the proportion of samples whose +class is the positive class (in each bin). .. figure:: ../auto_examples/calibration/images/sphx_glr_plot_compare_calibration_001.png :target: ../auto_examples/calibration/plot_compare_calibration.html @@ -34,177 +42,117 @@ with different biases per method: .. currentmodule:: sklearn.naive_bayes -* :class:`GaussianNB` tends to push probabilities to 0 or 1 (note the - counts in the histograms). This is mainly because it makes the assumption - that features are conditionally independent given the class, which is not - the case in this dataset which contains 2 redundant features. +:class:`GaussianNB` tends to push probabilities to 0 or 1 (note the counts +in the histograms). This is mainly because it makes the assumption that +features are conditionally independent given the class, which is not the +case in this dataset which contains 2 redundant features. .. currentmodule:: sklearn.ensemble -* :class:`RandomForestClassifier` shows the opposite behavior: the histograms - show peaks at approximately 0.2 and 0.9 probability, while probabilities close to - 0 or 1 are very rare. An explanation for this is given by Niculescu-Mizil - and Caruana [4]_: "Methods such as bagging and random forests that average - predictions from a base set of models can have difficulty making predictions - near 0 and 1 because variance in the underlying base models will bias - predictions that should be near zero or one away from these values. Because - predictions are restricted to the interval [0,1], errors caused by variance - tend to be one-sided near zero and one. For example, if a model should - predict p = 0 for a case, the only way bagging can achieve this is if all - bagged trees predict zero. If we add noise to the trees that bagging is - averaging over, this noise will cause some trees to predict values larger - than 0 for this case, thus moving the average prediction of the bagged - ensemble away from 0. We observe this effect most strongly with random - forests because the base-level trees trained with random forests have - relatively high variance due to feature subsetting." As a result, the - calibration curve also referred to as the reliability diagram (Wilks 1995 [5]_) shows a - characteristic sigmoid shape, indicating that the classifier could trust its - "intuition" more and return probabilities closer to 0 or 1 typically. +:class:`RandomForestClassifier` shows the opposite behavior: the histograms +show peaks at approximately 0.2 and 0.9 probability, while probabilities +close to 0 or 1 are very rare. An explanation for this is given by +Niculescu-Mizil and Caruana [1]_: "Methods such as bagging and random +forests that average predictions from a base set of models can have +difficulty making predictions near 0 and 1 because variance in the +underlying base models will bias predictions that should be near zero or one +away from these values. Because predictions are restricted to the interval +[0,1], errors caused by variance tend to be one-sided near zero and one. For +example, if a model should predict p = 0 for a case, the only way bagging +can achieve this is if all bagged trees predict zero. If we add noise to the +trees that bagging is averaging over, this noise will cause some trees to +predict values larger than 0 for this case, thus moving the average +prediction of the bagged ensemble away from 0. We observe this effect most +strongly with random forests because the base-level trees trained with +random forests have relatively high variance due to feature subsetting." As +a result, the calibration curve also referred to as the reliability diagram +(Wilks 1995 [2]_) shows a characteristic sigmoid shape, indicating that the +classifier could trust its "intuition" more and return probabilities closer +to 0 or 1 typically. .. currentmodule:: sklearn.svm -* Linear Support Vector Classification (:class:`LinearSVC`) shows an even more sigmoid curve - as the RandomForestClassifier, which is typical for maximum-margin methods - (compare Niculescu-Mizil and Caruana [4]_), which focus on hard samples - that are close to the decision boundary (the support vectors). - -.. currentmodule:: sklearn.calibration - -Two approaches for performing calibration of probabilistic predictions are -provided: a parametric approach based on Platt's sigmoid model and a -non-parametric approach based on isotonic regression (:mod:`sklearn.isotonic`). -Probability calibration should be done on new data not used for model fitting. -The class :class:`CalibratedClassifierCV` uses a cross-validation generator and -estimates for each split the model parameter on the train samples and the -calibration of the test samples. The probabilities predicted for the -folds are then averaged. Already fitted classifiers can be calibrated by -:class:`CalibratedClassifierCV` via the parameter cv="prefit". In this case, -the user has to take care manually that data for model fitting and calibration -are disjoint. - -The following images demonstrate the benefit of probability calibration. -The first image present a dataset with 2 classes and 3 blobs of -data. The blob in the middle contains random samples of each class. -The probability for the samples in this blob should be 0.5. - -.. figure:: ../auto_examples/calibration/images/sphx_glr_plot_calibration_001.png - :target: ../auto_examples/calibration/plot_calibration.html - :align: center - -The following image shows on the data above the estimated probability -using a Gaussian naive Bayes classifier without calibration, -with a sigmoid calibration and with a non-parametric isotonic -calibration. One can observe that the non-parametric model -provides the most accurate probability estimates for samples -in the middle, i.e., 0.5. - -.. figure:: ../auto_examples/calibration/images/sphx_glr_plot_calibration_002.png - :target: ../auto_examples/calibration/plot_calibration.html - :align: center - -.. currentmodule:: sklearn.metrics - -The following experiment is performed on an artificial dataset for binary -classification with 100,000 samples (1,000 of them are used for model fitting) -with 20 features. Of the 20 features, only 2 are informative and 10 are -redundant. The figure shows the estimated probabilities obtained with -logistic regression, a linear support-vector classifier (SVC), and linear SVC with -both isotonic calibration and sigmoid calibration. -The Brier score is a metric which is a combination of calibration loss and refinement loss, -:func:`brier_score_loss`, reported in the legend (the smaller the better). -Calibration loss is defined as the mean squared deviation from empirical probabilities -derived from the slope of ROC segments. Refinement loss can be defined as the expected -optimal loss as measured by the area under the optimal cost curve. - -.. figure:: ../auto_examples/calibration/images/sphx_glr_plot_calibration_curve_002.png - :target: ../auto_examples/calibration/plot_calibration_curve.html - :align: center +Linear Support Vector Classification (:class:`LinearSVC`) shows an even more +sigmoid curve as the RandomForestClassifier, which is typical for +maximum-margin methods (compare Niculescu-Mizil and Caruana [1]_), which +focus on hard samples that are close to the decision boundary (the support +vectors). -One can observe here that logistic regression is well calibrated as its curve is -nearly diagonal. Linear SVC's calibration curve or reliability diagram has a -sigmoid curve, which is typical for an under-confident classifier. In the case of -LinearSVC, this is caused by the margin property of the hinge loss, which lets -the model focus on hard samples that are close to the decision boundary -(the support vectors). Both kinds of calibration can fix this issue and yield -nearly identical results. The next figure shows the calibration curve of -Gaussian naive Bayes on the same data, with both kinds of calibration and also -without calibration. - -.. figure:: ../auto_examples/calibration/images/sphx_glr_plot_calibration_curve_001.png - :target: ../auto_examples/calibration/plot_calibration_curve.html - :align: center - -One can see that Gaussian naive Bayes performs very badly but does so in an -other way than linear SVC: While linear SVC exhibited a sigmoid calibration -curve, Gaussian naive Bayes' calibration curve has a transposed-sigmoid shape. -This is typical for an over-confident classifier. In this case, the classifier's -overconfidence is caused by the redundant features which violate the naive Bayes -assumption of feature-independence. - -Calibration of the probabilities of Gaussian naive Bayes with isotonic -regression can fix this issue as can be seen from the nearly diagonal -calibration curve. Sigmoid calibration also improves the brier score slightly, -albeit not as strongly as the non-parametric isotonic calibration. This is an -intrinsic limitation of sigmoid calibration, whose parametric form assumes a -sigmoid rather than a transposed-sigmoid curve. The non-parametric isotonic -calibration model, however, makes no such strong assumptions and can deal with -either shape, provided that there is sufficient calibration data. In general, -sigmoid calibration is preferable in cases where the calibration curve is sigmoid -and where there is limited calibration data, while isotonic calibration is -preferable for non-sigmoid calibration curves and in situations where large -amounts of data are available for calibration. +Calibrating a classifier +------------------------ .. currentmodule:: sklearn.calibration -:class:`CalibratedClassifierCV` can also deal with classification tasks that -involve more than two classes if the base estimator can do so. In this case, -the classifier is calibrated first for each class separately in an one-vs-rest -fashion. When predicting probabilities for unseen data, the calibrated -probabilities for each class are predicted separately. As those probabilities -do not necessarily sum to one, a postprocessing is performed to normalize them. - -The next image illustrates how sigmoid calibration changes predicted -probabilities for a 3-class classification problem. Illustrated is the standard -2-simplex, where the three corners correspond to the three classes. Arrows point -from the probability vectors predicted by an uncalibrated classifier to the -probability vectors predicted by the same classifier after sigmoid calibration -on a hold-out validation set. Colors indicate the true class of an instance -(red: class 1, green: class 2, blue: class 3). - -.. figure:: ../auto_examples/calibration/images/sphx_glr_plot_calibration_multiclass_001.png - :target: ../auto_examples/calibration/plot_calibration_multiclass.html - :align: center - -The base classifier is a random forest classifier with 25 base estimators -(trees). If this classifier is trained on all 800 training datapoints, it is -overly confident in its predictions and thus incurs a large log-loss. -Calibrating an identical classifier, which was trained on 600 datapoints, with -method='sigmoid' on the remaining 200 datapoints reduces the confidence of the -predictions, i.e., moves the probability vectors from the edges of the simplex -towards the center: - -.. figure:: ../auto_examples/calibration/images/sphx_glr_plot_calibration_multiclass_002.png - :target: ../auto_examples/calibration/plot_calibration_multiclass.html - :align: center - -This calibration results in a lower log-loss. Note that an alternative would -have been to increase the number of base estimators which would have resulted in -a similar decrease in log-loss. +Calibrating a classifier consists in fitting a regressor (called a +*calibrator*) that maps the output of the classifier (as given by +:term:`predict` or :term:`predict_proba`) to a calibrated probability in [0, +1]. Denoting the output of the classifier for a given sample by :math:`f_i`, +the calibrator tries to predict :math:`p(y_i = 1 | f_i)`. + +The samples that are used to train the calibrator should not be used to +train the target classifier. + +Usage +----- + +The :class:`CalibratedClassifierCV` class is used to calibrate a classifier. + +:class:`CalibratedClassifierCV` uses a cross-validation approach to fit both +the classifier and the regressor. For each of the k `(trainset, testset)` +couple, a classifier is trained on the train set, and its predictions on the +test set are used to fit a regressor. We end up with k +`(classifier, regressor)` couples where each regressor maps the output of +its corresponding classifier into [0, 1]. Each couple is exposed in the +`calibrated_classifiers_` attribute, where each entry is a calibrated +classifier with a :term:`predict_proba` method that outputs calibrated +probabilities. The output of :term:`predict_proba` for the main +:class:`CalibratedClassifierCV` instance corresponds to the average of the +predicted probabilities of the `k` estimators in the +`calibrated_classifiers_` list. The output of :term:`predict` is the class +that has the highest probability. + +The regressor that is used for calibration depends on the `method` +parameter. `'sigmoid'` corresponds to a parametric approach based on Platt's +logistic model [3]_, i.e. :math:`p(y_i = 1 | f_i)` is modeled as +:math:`\sigma(A f_i + B)` where :math:`\sigma` is the logistic function, and +:math:`A` and :math:`B` are real numbers to be determined when fitting the +regressor via maximum likelihood. `'isotonic'` will instead fit a +non-parametric isotonic regressor, which outputs a step-wise non-decreasing +function (see :mod:`sklearn.isotonic`). + +An already fitted classifier can be calibrated by setting `cv="prefit"`. In +this case, the data is only used to fit the regressor. It is up to the user +make sure that the data used for fitting the classifier is disjoint from the +data used for fitting the regressor. + +:class:`CalibratedClassifierCV` can calibrate probabilities in a multiclass +setting if the base estimator supports multiclass predictions. The classifier +is calibrated first for each class separately in a one-vs-rest fashion [4]_. +When predicting probabilities, the calibrated probabilities for each class +are predicted separately. As those probabilities do not necessarily sum to +one, a postprocessing is performed to normalize them. + +The :func:`sklearn.metrics.brier_score_loss` may be used to evaluate how +well a classifier is calibrated. + +.. topic:: Examples: + + * :ref:`sphx_glr_auto_examples_calibration_plot_calibration_curve.py` + * :ref:`sphx_glr_auto_examples_calibration_plot_calibration_multiclass.py` + * :ref:`sphx_glr_auto_examples_calibration_plot_calibration.py` + * :ref:`sphx_glr_auto_examples_calibration_plot_compare_calibration.py` .. topic:: References: - * Obtaining calibrated probability estimates from decision trees - and naive Bayesian classifiers, B. Zadrozny & C. Elkan, ICML 2001 - - * Transforming Classifier Scores into Accurate Multiclass - Probability Estimates, B. Zadrozny & C. Elkan, (KDD 2002) - - * Probabilistic Outputs for Support Vector Machines and Comparisons to - Regularized Likelihood Methods, J. Platt, (1999) - - .. [4] Predicting Good Probabilities with Supervised Learning, + .. [1] Predicting Good Probabilities with Supervised Learning, A. Niculescu-Mizil & R. Caruana, ICML 2005 - .. [5] On the combination of forecast probabilities for + .. [2] On the combination of forecast probabilities for consecutive precipitation periods. Wea. Forecasting, 5, 640–650., Wilks, D. S., 1990a + + .. [3] Probabilistic Outputs for Support Vector Machines and Comparisons + to Regularized Likelihood Methods, J. Platt, (1999) + + .. [4] Transforming Classifier Scores into Accurate Multiclass + Probability Estimates, B. Zadrozny & C. Elkan, (KDD 2002) diff --git a/doc/modules/classes.rst b/doc/modules/classes.rst index 53ce0c94ece9f..c138f51f6c06f 100644 --- a/doc/modules/classes.rst +++ b/doc/modules/classes.rst @@ -33,6 +33,7 @@ Base classes base.DensityMixin base.RegressorMixin base.TransformerMixin + feature_selection.SelectorMixin Functions --------- @@ -836,6 +837,21 @@ Any estimator using the Huber loss would also be robust to outliers, e.g. linear_model.RANSACRegressor linear_model.TheilSenRegressor +Generalized linear models (GLM) for regression +---------------------------------------------- + +These models allow for response variables to have error distributions other +than a normal distribution: + +.. autosummary:: + :toctree: generated/ + :template: class.rst + + linear_model.PoissonRegressor + linear_model.TweedieRegressor + linear_model.GammaRegressor + + Miscellaneous ------------- @@ -1082,6 +1098,7 @@ See the :ref:`visualizations` section of the user guide for further details. :toctree: generated/ :template: function.rst + metrics.plot_confusion_matrix metrics.plot_precision_recall_curve metrics.plot_roc_curve @@ -1089,6 +1106,7 @@ See the :ref:`visualizations` section of the user guide for further details. :toctree: generated/ :template: class.rst + metrics.ConfusionMatrixDisplay metrics.PrecisionRecallDisplay metrics.RocCurveDisplay @@ -1176,12 +1194,6 @@ Hyper-parameter optimizers model_selection.RandomizedSearchCV -.. autosummary:: - :toctree: generated/ - :template: function.rst - - model_selection.fit_grid_point - Model validation ---------------- @@ -1543,6 +1555,7 @@ Plotting utils.arrayfuncs.min_pos utils.as_float_array utils.assert_all_finite + utils.Bunch utils.check_X_y utils.check_array utils.check_scalar @@ -1603,30 +1616,12 @@ Utilities from joblib: Recently deprecated =================== -To be removed in 0.23 +To be removed in 0.24 --------------------- -.. autosummary:: - :toctree: generated/ - :template: deprecated_class.rst - - utils.Memory - utils.Parallel - .. autosummary:: :toctree: generated/ :template: deprecated_function.rst - utils.cpu_count - utils.delayed - metrics.calinski_harabaz_score - metrics.jaccard_similarity_score - linear_model.logistic_regression_path + model_selection.fit_grid_point utils.safe_indexing - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - ensemble.partial_dependence.partial_dependence - ensemble.partial_dependence.plot_partial_dependence diff --git a/doc/modules/clustering.rst b/doc/modules/clustering.rst index 93f87989ab233..4a4314fdbafee 100644 --- a/doc/modules/clustering.rst +++ b/doc/modules/clustering.rst @@ -205,23 +205,17 @@ computing cluster centers and values of inertia. For example, assigning a weight of 2 to a sample is equivalent to adding a duplicate of that sample to the dataset :math:`X`. -A parameter can be given to allow K-means to be run in parallel, called -``n_jobs``. Giving this parameter a positive value uses that many processors -(default: 1). A value of -1 uses all available processors, with -2 using one -less, and so on. Parallelization generally speeds up computation at the cost of -memory (in this case, multiple copies of centroids need to be stored, one for -each job). - -.. warning:: - - The parallel version of K-Means is broken on OS X when `numpy` uses the - `Accelerate` Framework. This is expected behavior: `Accelerate` can be called - after a fork but you need to execv the subprocess with the Python binary - (which multiprocessing does not do under posix). - K-means can be used for vector quantization. This is achieved using the transform method of a trained model of :class:`KMeans`. +Low-level parallelism +--------------------- + +:class:`KMeans` benefits from OpenMP based parallelism through Cython. Small +chunks of data (256 samples) are processed in parallel, which in addition +yields a low memory footprint. For more details on how to control the number of +threads, please refer to our :ref:`parallelism` notes. + .. topic:: Examples: * :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_assumptions.py`: Demonstrating when @@ -775,7 +769,7 @@ core sample, and is at least ``eps`` in distance from any core sample, is considered an outlier by the algorithm. While the parameter ``min_samples`` primarily controls how tolerant the -algorithm is towards noise (on noisy and large data sets it may be desiable +algorithm is towards noise (on noisy and large data sets it may be desirable to increase this parameter), the parameter ``eps`` is *crucial to choose appropriately* for the data set and distance function and usually cannot be left at the default value. It controls the local neighborhood of the points. @@ -1687,6 +1681,7 @@ Drawbacks Calinski-Harabasz Index ----------------------- + If the ground truth labels are not known, the Calinski-Harabasz index (:func:`sklearn.metrics.calinski_harabasz_score`) - also known as the Variance Ratio Criterion - can be used to evaluate the model, where a higher diff --git a/doc/modules/computing.rst b/doc/modules/computing.rst index 176b8e22fca1c..246085d436cde 100644 --- a/doc/modules/computing.rst +++ b/doc/modules/computing.rst @@ -529,7 +529,7 @@ Joblib-based parallelism ........................ When the underlying implementation uses joblib, the number of workers -(threads or processes) that are spawned in parallel can be controled via the +(threads or processes) that are spawned in parallel can be controlled via the ``n_jobs`` parameter. .. note:: @@ -666,7 +666,7 @@ Python runtime :working_memory: - the optimal size of temporary arrays used by some algoritms. + the optimal size of temporary arrays used by some algorithms. .. _environment_variable: diff --git a/doc/modules/density.rst b/doc/modules/density.rst index b6dbe5e49abbb..1070b9fbf9f1b 100644 --- a/doc/modules/density.rst +++ b/doc/modules/density.rst @@ -93,7 +93,7 @@ Given this kernel form, the density estimate at a point :math:`y` within a group of points :math:`x_i; i=1\cdots N` is given by: .. math:: - \rho_K(y) = \sum_{i=1}^{N} K((y - x_i) / h) + \rho_K(y) = \sum_{i=1}^{N} K(y - x_i; h) The bandwidth here acts as a smoothing parameter, controlling the tradeoff between bias and variance in the result. A large bandwidth leads to a very diff --git a/doc/modules/ensemble.rst b/doc/modules/ensemble.rst index ec7b337a20593..b7c0e49f9c477 100644 --- a/doc/modules/ensemble.rst +++ b/doc/modules/ensemble.rst @@ -279,6 +279,19 @@ for feature selection. This is known as the mean decrease in impurity, or MDI. Refer to [L2014]_ for more information on MDI and feature importance evaluation with Random Forests. +.. warning:: + + The impurity-based feature importances computed on tree-based models suffer + from two flaws that can lead to misleading conclusions. First they are + computed on statistics derived from the training dataset and therefore **do + not necessarily inform us on which features are most important to make good + predictions on held-out dataset**. Secondly, **they favor high cardinality + features**, that is features with many unique values. + :ref:`permutation_importance` is an alternative to impurity-based feature + importance that does not suffer from these flaws. These two methods of + obtaining feature importance are explored in: + :ref:`sphx_glr_auto_examples_inspection_plot_permutation_importance.py`. + The following example shows a color-coded representation of the relative importances of each individual pixel for a face recognition task using a :class:`ExtraTreesClassifier` model. @@ -322,8 +335,9 @@ trees and the maximum depth per tree. For each tree in the ensemble, the coding contains one entry of one. The size of the coding is at most ``n_estimators * 2 ** max_depth``, the maximum number of leaves in the forest. -As neighboring data points are more likely to lie within the same leaf of a tree, -the transformation performs an implicit, non-parametric density estimation. +As neighboring data points are more likely to lie within the same leaf of a +tree, the transformation performs an implicit, non-parametric density +estimation. .. topic:: Examples: @@ -475,6 +489,10 @@ trees. in this setting. +The usage and the parameters of :class:`GradientBoostingClassifier` and +:class:`GradientBoostingRegressor` are described below. The 2 most important +parameters of these estimators are `n_estimators` and `learning_rate`. + Classification --------------- @@ -495,7 +513,13 @@ with 100 decision stumps as weak learners:: >>> clf.score(X_test, y_test) 0.913... -The number of weak learners (i.e. regression trees) is controlled by the parameter ``n_estimators``; :ref:`The size of each tree ` can be controlled either by setting the tree depth via ``max_depth`` or by setting the number of leaf nodes via ``max_leaf_nodes``. The ``learning_rate`` is a hyper-parameter in the range (0.0, 1.0] that controls overfitting via :ref:`shrinkage ` . +The number of weak learners (i.e. regression trees) is controlled by the +parameter ``n_estimators``; :ref:`The size of each tree +` can be controlled either by setting the tree +depth via ``max_depth`` or by setting the number of leaf nodes via +``max_leaf_nodes``. The ``learning_rate`` is a hyper-parameter in the range +(0.0, 1.0] that controls overfitting via :ref:`shrinkage +` . .. note:: @@ -540,8 +564,8 @@ of the gradient boosting model. The test error at each iterations can be obtaine via the :meth:`~GradientBoostingRegressor.staged_predict` method which returns a generator that yields the predictions at each stage. Plots like these can be used to determine the optimal number of trees (i.e. ``n_estimators``) by early stopping. -The plot on the right shows the feature importances which can be obtained via -the ``feature_importances_`` property. +The plot on the right shows the impurity-based feature importances which can be +obtained via the ``feature_importances_`` property. .. figure:: ../auto_examples/ensemble/images/sphx_glr_plot_gradient_boosting_regression_001.png :target: ../auto_examples/ensemble/plot_gradient_boosting_regression.html @@ -601,65 +625,118 @@ chapter on gradient boosting in [F2001]_ and is related to the parameter Mathematical formulation ------------------------- -GBRT considers additive models of the following form: +We first present GBRT for regression, and then detail the classification +case. + +Regression +^^^^^^^^^^ + +GBRT regressors are additive models whose prediction :math:`y_i` for a +given input :math:`x_i` is of the following form: .. math:: - F(x) = \sum_{m=1}^{M} \gamma_m h_m(x) + \hat{y_i} = F_M(x_i) = \sum_{m=1}^{M} h_m(x_i) -where :math:`h_m(x)` are the basis functions which are usually called -*weak learners* in the context of boosting. Gradient Tree Boosting -uses :ref:`decision trees ` of fixed size as weak -learners. Decision trees have a number of abilities that make them -valuable for boosting, namely the ability to handle data of mixed type -and the ability to model complex functions. +where the :math:`h_m` are estimators called *weak learners* in the context +of boosting. Gradient Tree Boosting uses :ref:`decision tree regressors +` of fixed size as weak learners. The constant M corresponds to the +`n_estimators` parameter. -Similar to other boosting algorithms, GBRT builds the additive model in -a greedy fashion: +Similar to other boosting algorithms, a GBRT is built in a greedy fashion: .. math:: - F_m(x) = F_{m-1}(x) + \gamma_m h_m(x), + F_m(x) = F_{m-1}(x) + h_m(x), -where the newly added tree :math:`h_m` tries to minimize the loss :math:`L`, -given the previous ensemble :math:`F_{m-1}`: +where the newly added tree :math:`h_m` is fitted in order to minimize a sum +of losses :math:`L_m`, given the previous ensemble :math:`F_{m-1}`: .. math:: - h_m = \arg\min_{h} \sum_{i=1}^{n} L(y_i, - F_{m-1}(x_i) + h(x_i)). + h_m = \arg\min_{h} L_m = \arg\min_{h} \sum_{i=1}^{n} + l(y_i, F_{m-1}(x_i) + h(x_i)), -The initial model :math:`F_{0}` is problem specific, for least-squares -regression one usually chooses the mean of the target values. +where :math:`l(y_i, F(x_i))` is defined by the `loss` parameter, detailed +in the next section. -.. note:: The initial model can also be specified via the ``init`` - argument. The passed object has to implement ``fit`` and ``predict``. +By default, the initial model :math:`F_{0}` is chosen as the constant that +minimizes the loss: for a least-squares loss, this is the empirical mean of +the target values. The initial model can also be specified via the ``init`` +argument. -Gradient Boosting attempts to solve this minimization problem -numerically via steepest descent: The steepest descent direction is -the negative gradient of the loss function evaluated at the current -model :math:`F_{m-1}` which can be calculated for any differentiable -loss function: +Using a first-order Taylor approximation, the value of :math:`l` can be +approximated as follows: .. math:: - F_m(x) = F_{m-1}(x) - \gamma_m \sum_{i=1}^{n} \nabla_F L(y_i, - F_{m-1}(x_i)) + l(y_i, F_{m-1}(x_i) + h_m(x_i)) \approx + l(y_i, F_{m-1}(x_i)) + + h_m(x_i) + \left[ \frac{\partial l(y_i, F(x_i))}{\partial F(x_i)} \right]_{F=F_{m - 1}}. + +.. note:: + + Briefly, a first-order Taylor approximation says that + :math:`l(z) \approx l(a) + (z - a) \frac{\partial l(a)}{\partial a}`. + Here, :math:`z` corresponds to :math:`F_{m - 1}(x_i) + h_m(x_i)`, and + :math:`a` corresponds to :math:`F_{m-1}(x_i)` -Where the step length :math:`\gamma_m` is chosen using line search: +The quantity :math:`\left[ \frac{\partial l(y_i, F(x_i))}{\partial F(x_i)} +\right]_{F=F_{m - 1}}` is the derivative of the loss with respect to its +second parameter, evaluated at :math:`F_{m-1}(x)`. It is easy to compute for +any given :math:`F_{m - 1}(x_i)` in a closed form since the loss is +differentiable. We will denote it by :math:`g_i`. + +Removing the constant terms, we have: .. math:: - \gamma_m = \arg\min_{\gamma} \sum_{i=1}^{n} L(y_i, F_{m-1}(x_i) - - \gamma \frac{\partial L(y_i, F_{m-1}(x_i))}{\partial F_{m-1}(x_i)}) + h_m \approx \arg\min_{h} \sum_{i=1}^{n} h(x_i) g_i + +This is minimized if :math:`h(x_i)` is fitted to predict a value that is +proportional to the negative gradient :math:`-g_i`. Therefore, at each +iteration, **the estimator** :math:`h_m` **is fitted to predict the negative +gradients of the samples**. The gradients are updated at each iteration. +This can be considered as some kind of gradient descent in a functional +space. + +.. note:: + + For some losses, e.g. the least absolute deviation (LAD) where the gradients + are :math:`\pm 1`, the values predicted by a fitted :math:`h_m` are not + accurate enough: the tree can only output integer values. As a result, the + leaves values of the tree :math:`h_m` are modified once the tree is + fitted, such that the leaves values minimize the loss :math:`L_m`. The + update is loss-dependent: for the LAD loss, the value of a leaf is updated + to the median of the samples in that leaf. + +Classification +^^^^^^^^^^^^^^ -The algorithms for regression and classification -only differ in the concrete loss function used. +Gradient boosting for classification is very similar to the regression case. +However, the sum of the trees :math:`F_M(x_i) = \sum_m h_m(x_i)` is not +homogeneous to a prediction: it cannot be a class, since the trees predict +continuous values. + +The mapping from the value :math:`F_M(x_i)` to a class or a probability is +loss-dependent. For the deviance (or log-loss), the probability that +:math:`x_i` belongs to the positive class is modeled as :math:`p(y_i = 1 | +x_i) = \sigma(F_M(x_i))` where :math:`\sigma` is the sigmoid function. + +For multiclass classification, K trees (for K classes) are built at each of +the :math:`M` iterations. The probability that :math:`x_i` belongs to class +k is modeled as a softmax of the :math:`F_{M,k}(x_i)` values. + +Note that even for a classification task, the :math:`h_m` sub-estimator is +still a regressor, not a classifier. This is because the sub-estimators are +trained to predict (negative) *gradients*, which are always continuous +quantities. .. _gradient_boosting_loss: Loss Functions -............... +-------------- The following loss functions are supported and can be specified using the parameter ``loss``: @@ -699,20 +776,17 @@ the parameter ``loss``: examples than ``'deviance'``; can only be used for binary classification. -Regularization ----------------- - .. _gradient_boosting_shrinkage: -Shrinkage -.......... +Shrinkage via learning rate +--------------------------- [F2001]_ proposed a simple regularization strategy that scales -the contribution of each weak learner by a factor :math:`\nu`: +the contribution of each weak learner by a constant factor :math:`\nu`: .. math:: - F_m(x) = F_{m-1}(x) + \nu \gamma_m h_m(x) + F_m(x) = F_{m-1}(x) + \nu h_m(x) The parameter :math:`\nu` is also called the **learning rate** because it scales the step length the gradient descent procedure; it can @@ -729,7 +803,7 @@ stopping. For a more detailed discussion of the interaction between ``learning_rate`` and ``n_estimators`` see [R2007]_. Subsampling -............ +----------- [F1999]_ proposed stochastic gradient boosting, which combines gradient boosting with bootstrap averaging (bagging). At each iteration @@ -773,8 +847,8 @@ is too time consuming. * :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_oob.py` * :ref:`sphx_glr_auto_examples_ensemble_plot_ensemble_oob.py` -Interpretation --------------- +Interpretation with feature importance +-------------------------------------- Individual decision trees can be interpreted easily by simply visualizing the tree structure. Gradient boosting models, however, @@ -783,9 +857,6 @@ interpreted by visual inspection of the individual trees. Fortunately, a number of techniques have been proposed to summarize and interpret gradient boosting models. -Feature importance -.................. - Often features do not contribute equally to predict the target response; in many situations the majority of the features are in fact irrelevant. @@ -798,7 +869,7 @@ appropriate split points. This information can be used to measure the importance of each feature; the basic idea is: the more often a feature is used in the split points of a tree the more important that feature is. This notion of importance can be extended to decision tree -ensembles by simply averaging the feature importance of each tree (see +ensembles by simply averaging the impurity-based feature importance of each tree (see :ref:`random_forest_feature_importance` for more details). The feature importance scores of a fit gradient boosting model can be @@ -813,6 +884,10 @@ accessed via the ``feature_importances_`` property:: >>> clf.feature_importances_ array([0.10..., 0.10..., 0.11..., ... +Note that this computation of feature importance is based on entropy, and it +is distinct from :func:`sklearn.inspection.permutation_importance` which is +based on permutation of the features. + .. topic:: Examples: * :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_regression.py` @@ -842,8 +917,7 @@ leverage integer-based data structures (histograms) instead of relying on sorted continuous values when building the trees. The API of these estimators is slightly different, and some of the features from :class:`GradientBoostingClassifier` and :class:`GradientBoostingRegressor` -are not yet supported: in particular sample weights, and some loss -functions. +are not yet supported, for instance some loss functions. These estimators are still **experimental**: their predictions and their API might change without any deprecation cycle. To use them, you @@ -895,12 +969,14 @@ generally recommended to use as many bins as possible, which is the default. The ``l2_regularization`` parameter is a regularizer on the loss function and corresponds to :math:`\lambda` in equation (2) of [XGBoost]_. -Note that **early-stopping is enabled by default**. The early-stopping -behaviour is controlled via the ``scoring``, ``validation_fraction``, +Note that **early-stopping is enabled by default if the number of samples is +larger than 10,000**. The early-stopping behaviour is controlled via the +``early-stopping``, ``scoring``, ``validation_fraction``, ``n_iter_no_change``, and ``tol`` parameters. It is possible to early-stop -using an arbitrary :term:`scorer`, or just the training or validation loss. By -default, early-stopping is performed using the default :term:`scorer` of -the estimator on a validation set. +using an arbitrary :term:`scorer`, or just the training or validation loss. +Note that for technical reasons, using a scorer is significantly slower than +using the loss. By default, early-stopping is performed if there are at least +10,000 samples in the training set, using the validation loss. Missing values support ---------------------- @@ -941,6 +1017,39 @@ If no missing values were encountered for a given feature during training, then samples with missing values are mapped to whichever child has the most samples. +Sample weight support +--------------------- + +:class:`HistGradientBoostingClassifier` and +:class:`HistGradientBoostingRegressor` sample support weights during +:term:`fit`. + +The following toy example demonstrates how the model ignores the samples with +zero sample weights: + + >>> X = [[1, 0], + ... [1, 0], + ... [1, 0], + ... [0, 1]] + >>> y = [0, 0, 1, 0] + >>> # ignore the first 2 training samples by setting their weight to 0 + >>> sample_weight = [0, 0, 1, 1] + >>> gb = HistGradientBoostingClassifier(min_samples_leaf=1) + >>> gb.fit(X, y, sample_weight=sample_weight) + HistGradientBoostingClassifier(...) + >>> gb.predict([[1, 0]]) + array([1]) + >>> gb.predict_proba([[1, 0]])[0, 1] + 0.99... + +As you can see, the `[1, 0]` is comfortably classified as `1` since the first +two samples are ignored due to their sample weights. + +Implementation detail: taking sample weights into account amounts to +multiplying the gradients (and the hessians) by the sample weights. Note that +the binning stage (specifically the quantiles computation) does not take the +weights into account. + Low-level parallelism --------------------- @@ -1041,7 +1150,7 @@ based on the ascending sort order. E.g., in the following scenario the class label 1 will be assigned to the sample. Usage -..... +----- The following example shows how to fit the majority rule classifier:: @@ -1158,7 +1267,7 @@ hyperparameters of the individual estimators:: >>> grid = grid.fit(iris.data, iris.target) Usage -..... +----- In order to predict the class labels based on the predicted class-probabilities (scikit-learn estimators in the VotingClassifier @@ -1322,7 +1431,7 @@ computationally expensive. StackingRegressor(...) >>> print('R2 score: {:.2f}' ... .format(multi_layer_regressor.score(X_test, y_test))) - R2 score: 0.82 + R2 score: 0.83 .. topic:: References diff --git a/doc/modules/feature_selection.rst b/doc/modules/feature_selection.rst index 0de9b0b43f9fc..6a319cfb94336 100644 --- a/doc/modules/feature_selection.rst +++ b/doc/modules/feature_selection.rst @@ -158,8 +158,8 @@ For examples on how it is to be used refer to the sections below. .. topic:: Examples - * :ref:`sphx_glr_auto_examples_feature_selection_plot_select_from_model_boston.py`: Selecting the two - most important features from the Boston dataset without knowing the + * :ref:`sphx_glr_auto_examples_feature_selection_plot_select_from_model_diabetes.py`: Selecting the two + most important features from the diabetes dataset without knowing the threshold beforehand. .. _l1_feature_selection: @@ -232,7 +232,7 @@ Tree-based feature selection Tree-based estimators (see the :mod:`sklearn.tree` module and forest of trees in the :mod:`sklearn.ensemble` module) can be used to compute -feature importances, which in turn can be used to discard irrelevant +impurity-based feature importances, which in turn can be used to discard irrelevant features (when coupled with the :class:`sklearn.feature_selection.SelectFromModel` meta-transformer):: diff --git a/doc/modules/gaussian_process.rst b/doc/modules/gaussian_process.rst index 9ff7dbf09f3db..668178c3e23a3 100644 --- a/doc/modules/gaussian_process.rst +++ b/doc/modules/gaussian_process.rst @@ -365,7 +365,8 @@ translations in the input space, while non-stationary kernels depend also on the specific values of the datapoints. Stationary kernels can further be subdivided into isotropic and anisotropic kernels, where isotropic kernels are also invariant to rotations in the input space. For more details, we refer to -Chapter 4 of [RW2006]_. +Chapter 4 of [RW2006]_. For guidance on how to best combine different kernels, +we refer to [Duv2014]_. Gaussian Process Kernel API --------------------------- @@ -470,13 +471,16 @@ It is defined as: Kernel operators ---------------- Kernel operators take one or two base kernels and combine them into a new -kernel. The :class:`Sum` kernel takes two kernels :math:`k1` and :math:`k2` -and combines them via :math:`k_{sum}(X, Y) = k1(X, Y) + k2(X, Y)`. -The :class:`Product` kernel takes two kernels :math:`k1` and :math:`k2` -and combines them via :math:`k_{product}(X, Y) = k1(X, Y) * k2(X, Y)`. +kernel. The :class:`Sum` kernel takes two kernels :math:`k_1` and :math:`k_2` +and combines them via :math:`k_{sum}(X, Y) = k_1(X, Y) + k_2(X, Y)`. +The :class:`Product` kernel takes two kernels :math:`k_1` and :math:`k_2` +and combines them via :math:`k_{product}(X, Y) = k_1(X, Y) * k_2(X, Y)`. The :class:`Exponentiation` kernel takes one base kernel and a scalar parameter -:math:`exponent` and combines them via -:math:`k_{exp}(X, Y) = k(X, Y)^\text{exponent}`. +:math:`p` and combines them via +:math:`k_{exp}(X, Y) = k(X, Y)^p`. +Note that magic methods ``__add__``, ``__mul___`` and ``__pow__`` are +overridden on the Kernel objects, so one can use e.g. ``RBF() + RBF()`` as +a shortcut for ``Sum(RBF(), RBF())``. Radial-basis function (RBF) kernel ---------------------------------- @@ -487,8 +491,9 @@ number of dimensions as the inputs :math:`x` (anisotropic variant of the kernel) The kernel is given by: .. math:: - k(x_i, x_j) = \text{exp}\left(-\frac{1}{2} d(x_i / l, x_j / l)^2\right) + k(x_i, x_j) = \text{exp}\left(- \frac{d(x_i, x_j)^2}{2l^2} \right) +where :math:`d(\cdot, \cdot)` is the Euclidean distance. This kernel is infinitely differentiable, which implies that GPs with this kernel as covariance function have mean square derivatives of all orders, and are thus very smooth. The prior and posterior of a GP resulting from an RBF kernel are shown in @@ -507,24 +512,25 @@ the smoothness of the resulting function. It is parameterized by a length-scale .. math:: - k(x_i, x_j) = \sigma^2\frac{1}{\Gamma(\nu)2^{\nu-1}}\Bigg(\gamma\sqrt{2\nu} d(x_i / l, x_j / l)\Bigg)^\nu K_\nu\Bigg(\gamma\sqrt{2\nu} d(x_i / l, x_j / l)\Bigg), + k(x_i, x_j) = \frac{1}{\Gamma(\nu)2^{\nu-1}}\Bigg(\frac{\sqrt{2\nu}}{l} d(x_i , x_j )\Bigg)^\nu K_\nu\Bigg(\frac{\sqrt{2\nu}}{l} d(x_i , x_j )\Bigg), +where :math:`d(\cdot,\cdot)` is the Euclidean distance, :math:`K_\nu(\cdot)` is a modified Bessel function and :math:`\Gamma(\cdot)` is the gamma function. As :math:`\nu\rightarrow\infty`, the Matérn kernel converges to the RBF kernel. When :math:`\nu = 1/2`, the Matérn kernel becomes identical to the absolute exponential kernel, i.e., .. math:: - k(x_i, x_j) = \sigma^2 \exp \Bigg(-\gamma d(x_i / l, x_j / l) \Bigg) \quad \quad \nu= \tfrac{1}{2} + k(x_i, x_j) = \exp \Bigg(- \frac{1}{l} d(x_i , x_j ) \Bigg) \quad \quad \nu= \tfrac{1}{2} In particular, :math:`\nu = 3/2`: .. math:: - k(x_i, x_j) = \sigma^2 \Bigg(1 + \gamma \sqrt{3} d(x_i / l, x_j / l)\Bigg) \exp \Bigg(-\gamma \sqrt{3}d(x_i / l, x_j / l) \Bigg) \quad \quad \nu= \tfrac{3}{2} + k(x_i, x_j) = \Bigg(1 + \frac{\sqrt{3}}{l} d(x_i , x_j )\Bigg) \exp \Bigg(-\frac{\sqrt{3}}{l} d(x_i , x_j ) \Bigg) \quad \quad \nu= \tfrac{3}{2} and :math:`\nu = 5/2`: .. math:: - k(x_i, x_j) = \sigma^2 \Bigg(1 + \gamma \sqrt{5}d(x_i / l, x_j / l) +\frac{5}{3} \gamma^2d(x_i / l, x_j / l)^2 \Bigg) \exp \Bigg(-\gamma \sqrt{5}d(x_i / l, x_j / l) \Bigg) \quad \quad \nu= \tfrac{5}{2} + k(x_i, x_j) = \Bigg(1 + \frac{\sqrt{5}}{l} d(x_i , x_j ) +\frac{5}{3l} d(x_i , x_j )^2 \Bigg) \exp \Bigg(-\frac{\sqrt{5}}{l} d(x_i , x_j ) \Bigg) \quad \quad \nu= \tfrac{5}{2} are popular choices for learning functions that are not infinitely differentiable (as assumed by the RBF kernel) but at least once (:math:`\nu = @@ -570,7 +576,7 @@ It is parameterized by a length-scale parameter :math:`l>0` and a periodicity pa The kernel is given by: .. math:: - k(x_i, x_j) = \text{exp}\left(-2 \left(\text{sin}(\pi / p * d(x_i, x_j)) / l\right)^2\right) + k(x_i, x_j) = \text{exp}\left(- \frac{ 2\sin^2(\pi d(x_i, x_j) / p) }{ l^ 2} \right) The prior and posterior of a GP resulting from an ExpSineSquared kernel are shown in the following figure: @@ -604,4 +610,6 @@ References .. [RW2006] Carl Eduard Rasmussen and Christopher K.I. Williams, "Gaussian Processes for Machine Learning", MIT Press 2006, Link to an official complete PDF version of the book `here `_ . +.. [Duv2014] David Duvenaud, "The Kernel Cookbook: Advice on Covariance functions", 2014, `Link `_ . + .. currentmodule:: sklearn.gaussian_process diff --git a/doc/modules/glm_data/poisson_gamma_tweedie_distributions.png b/doc/modules/glm_data/poisson_gamma_tweedie_distributions.png new file mode 100644 index 0000000000000..3b95b724a6623 Binary files /dev/null and b/doc/modules/glm_data/poisson_gamma_tweedie_distributions.png differ diff --git a/doc/modules/isotonic.rst b/doc/modules/isotonic.rst index 097da60584383..1f54dcfa50bad 100644 --- a/doc/modules/isotonic.rst +++ b/doc/modules/isotonic.rst @@ -6,17 +6,27 @@ Isotonic regression .. currentmodule:: sklearn.isotonic -The class :class:`IsotonicRegression` fits a non-decreasing function to data. -It solves the following problem: +The class :class:`IsotonicRegression` fits a non-decreasing real function to +1-dimensional data. It solves the following problem: minimize :math:`\sum_i w_i (y_i - \hat{y}_i)^2` - subject to :math:`\hat{y}_{min} = \hat{y}_1 \le \hat{y}_2 ... \le \hat{y}_n = \hat{y}_{max}` + subject to :math:`\hat{y}_i \le \hat{y}_j` whenever :math:`X_i \le X_j`, -where each :math:`w_i` is strictly positive and each :math:`y_i` is an -arbitrary real number. It yields the vector which is composed of non-decreasing -elements the closest in terms of mean squared error. In practice this list -of elements forms a function that is piecewise linear. +where the weights :math:`w_i` are strictly positive, and both `X` and `y` are +arbitrary real quantities. + +The `increasing` parameter changes the constraint to +:math:`\hat{y}_i \ge \hat{y}_j` whenever :math:`X_i \le X_j`. Setting it to +'auto' will automatically choose the constraint based on `Spearman's rank +correlation coefficient +`_. + +:class:`IsotonicRegression` produces a series of predictions +:math:`\hat{y}_i` for the training data which are the closest to the targets +:math:`y` in terms of mean squared error. These predictions are interpolated +for predicting to unseen data. The predictions of :class:`IsotonicRegression` +thus form a function that is piecewise linear: .. figure:: ../auto_examples/images/sphx_glr_plot_isotonic_regression_001.png :target: ../auto_examples/plot_isotonic_regression.html diff --git a/doc/modules/kernel_ridge.rst b/doc/modules/kernel_ridge.rst index 3d032b52bb309..a67733b1ca5a5 100644 --- a/doc/modules/kernel_ridge.rst +++ b/doc/modules/kernel_ridge.rst @@ -7,44 +7,49 @@ Kernel ridge regression .. currentmodule:: sklearn.kernel_ridge Kernel ridge regression (KRR) [M2012]_ combines :ref:`ridge_regression` -(linear least squares with l2-norm regularization) with the kernel trick. It -thus learns a linear function in the space induced by the respective kernel and -the data. For non-linear kernels, this corresponds to a non-linear -function in the original space. +(linear least squares with l2-norm regularization) with the `kernel trick +`_. It thus learns a linear +function in the space induced by the respective kernel and the data. For +non-linear kernels, this corresponds to a non-linear function in the original +space. The form of the model learned by :class:`KernelRidge` is identical to support -vector regression (:class:`SVR`). However, different loss functions are used: -KRR uses squared error loss while support vector regression uses -:math:`\epsilon`-insensitive loss, both combined with l2 regularization. In -contrast to :class:`SVR`, fitting :class:`KernelRidge` can be done in -closed-form and is typically faster for medium-sized datasets. On the other -hand, the learned model is non-sparse and thus slower than SVR, which learns -a sparse model for :math:`\epsilon > 0`, at prediction-time. - -The following figure compares :class:`KernelRidge` and :class:`SVR` on -an artificial dataset, which consists of a sinusoidal target function and -strong noise added to every fifth datapoint. The learned model of -:class:`KernelRidge` and :class:`SVR` is plotted, where both -complexity/regularization and bandwidth of the RBF kernel have been optimized -using grid-search. The learned functions are very similar; however, fitting -:class:`KernelRidge` is approx. seven times faster than fitting :class:`SVR` -(both with grid-search). However, prediction of 100000 target values is more -than three times faster with SVR since it has learned a sparse model using only -approx. 1/3 of the 100 training datapoints as support vectors. +vector regression (:class:`~sklearn.svm.SVR`). However, different loss +functions are used: KRR uses squared error loss while support vector +regression uses :math:`\epsilon`-insensitive loss, both combined with l2 +regularization. In contrast to :class:`~sklearn.svm.SVR`, fitting +:class:`KernelRidge` can be done in closed-form and is typically faster for +medium-sized datasets. On the other hand, the learned model is non-sparse and +thus slower than :class:`~sklearn.svm.SVR`, which learns a sparse model for +:math:`\epsilon > 0`, at prediction-time. + +The following figure compares :class:`KernelRidge` and +:class:`~sklearn.svm.SVR` on an artificial dataset, which consists of a +sinusoidal target function and strong noise added to every fifth datapoint. +The learned model of :class:`KernelRidge` and :class:`~sklearn.svm.SVR` is +plotted, where both complexity/regularization and bandwidth of the RBF kernel +have been optimized using grid-search. The learned functions are very +similar; however, fitting :class:`KernelRidge` is approximately seven times +faster than fitting :class:`~sklearn.svm.SVR` (both with grid-search). +However, prediction of 100000 target values is more than three times faster +with :class:`~sklearn.svm.SVR` since it has learned a sparse model using only +approximately 1/3 of the 100 training datapoints as support vectors. .. figure:: ../auto_examples/images/sphx_glr_plot_kernel_ridge_regression_001.png :target: ../auto_examples/plot_kernel_ridge_regression.html :align: center The next figure compares the time for fitting and prediction of -:class:`KernelRidge` and :class:`SVR` for different sizes of the training set. -Fitting :class:`KernelRidge` is faster than :class:`SVR` for medium-sized -training sets (less than 1000 samples); however, for larger training sets -:class:`SVR` scales better. With regard to prediction time, :class:`SVR` is -faster than :class:`KernelRidge` for all sizes of the training set because of -the learned sparse solution. Note that the degree of sparsity and thus the -prediction time depends on the parameters :math:`\epsilon` and :math:`C` of the -:class:`SVR`; :math:`\epsilon = 0` would correspond to a dense model. +:class:`KernelRidge` and :class:`~sklearn.svm.SVR` for different sizes of the +training set. Fitting :class:`KernelRidge` is faster than +:class:`~sklearn.svm.SVR` for medium-sized training sets (less than 1000 +samples); however, for larger training sets :class:`~sklearn.svm.SVR` scales +better. With regard to prediction time, :class:`~sklearn.svm.SVR` is faster +than :class:`KernelRidge` for all sizes of the training set because of the +learned sparse solution. Note that the degree of sparsity and thus the +prediction time depends on the parameters :math:`\epsilon` and :math:`C` of +the :class:`~sklearn.svm.SVR`; :math:`\epsilon = 0` would correspond to a +dense model. .. figure:: ../auto_examples/images/sphx_glr_plot_kernel_ridge_regression_002.png :target: ../auto_examples/plot_kernel_ridge_regression.html diff --git a/doc/modules/linear_model.rst b/doc/modules/linear_model.rst index 19205385f311b..fc5f254035a53 100644 --- a/doc/modules/linear_model.rst +++ b/doc/modules/linear_model.rst @@ -556,13 +556,13 @@ orthogonal matching pursuit can approximate the optimum solution vector with a fixed number of non-zero elements: .. math:: - \underset{\gamma}{\operatorname{arg\,min\,}} ||y - X\gamma||_2^2 \text{ subject to } ||\gamma||_0 \leq n_{\text{nonzero\_coefs}} + \underset{w}{\operatorname{arg\,min\,}} ||y - Xw||_2^2 \text{ subject to } ||w||_0 \leq n_{\text{nonzero\_coefs}} Alternatively, orthogonal matching pursuit can target a specific error instead of a specific number of non-zero coefficients. This can be expressed as: .. math:: - \underset{\gamma}{\operatorname{arg\,min\,}} ||\gamma||_0 \text{ subject to } ||y-X\gamma||_2^2 \leq \text{tol} + \underset{w}{\operatorname{arg\,min\,}} ||w||_0 \text{ subject to } ||y-Xw||_2^2 \leq \text{tol} OMP is based on a greedy algorithm that includes at each step the atom most @@ -906,7 +906,7 @@ with 'log' loss, which might be even faster but requires more tuning. It is possible to obtain the p-values and confidence intervals for coefficients in cases of regression without penalization. The `statsmodels package ` natively supports this. - Within sklearn, one could use bootstrapping instead as well. + Within sklearn, one could use bootstrapping instead as well. :class:`LogisticRegressionCV` implements Logistic Regression with built-in @@ -928,6 +928,149 @@ to warm-starting (see :term:`Glossary `). .. [9] `"Performance Evaluation of Lbfgs vs other solvers" `_ +.. _Generalized_linear_regression: + +Generalized Linear Regression +============================= + +Generalized Linear Models (GLM) extend linear models in two ways +[10]_. First, the predicted values :math:`\hat{y}` are linked to a linear +combination of the input variables :math:`X` via an inverse link function +:math:`h` as + +.. math:: \hat{y}(w, X) = h(Xw). + +Secondly, the squared loss function is replaced by the unit deviance +:math:`d` of a distribution in the exponential family (or more precisely, a +reproductive exponential dispersion model (EDM) [11]_). + +The minimization problem becomes: + +.. math:: \min_{w} \frac{1}{2 n_{\text{samples}}} \sum_i d(y_i, \hat{y}_i) + \frac{\alpha}{2} ||w||_2, + +where :math:`\alpha` is the L2 regularization penalty. When sample weights are +provided, the average becomes a weighted average. + +The following table lists some specific EDMs and their unit deviance (all of +these are instances of the Tweedie family): + +================= =============================== ============================================ +Distribution Target Domain Unit Deviance :math:`d(y, \hat{y})` +================= =============================== ============================================ +Normal :math:`y \in (-\infty, \infty)` :math:`(y-\hat{y})^2` +Poisson :math:`y \in [0, \infty)` :math:`2(y\log\frac{y}{\hat{y}}-y+\hat{y})` +Gamma :math:`y \in (0, \infty)` :math:`2(\log\frac{\hat{y}}{y}+\frac{y}{\hat{y}}-1)` +Inverse Gaussian :math:`y \in (0, \infty)` :math:`\frac{(y-\hat{y})^2}{y\hat{y}^2}` +================= =============================== ============================================ + +The Probability Density Functions (PDF) of these distributions are illustrated +in the following figure, + +.. figure:: ./glm_data/poisson_gamma_tweedie_distributions.png + :align: center + :scale: 100% + + PDF of a random variable Y following Poisson, Tweedie (power=1.5) and Gamma + distributions with different mean values (:math:`\mu`). Observe the point + mass at :math:`Y=0` for the Poisson distribution and the Tweedie (power=1.5) + distribution, but not for the Gamma distribution which has a strictly + positive target domain. + +The choice of the distribution depends on the problem at hand: + +* If the target values :math:`y` are counts (non-negative integer valued) or + relative frequencies (non-negative), you might use a Poisson deviance + with log-link. +* If the target values are positive valued and skewed, you might try a + Gamma deviance with log-link. +* If the target values seem to be heavier tailed than a Gamma distribution, + you might try an Inverse Gaussian deviance (or even higher variance powers + of the Tweedie family). + + +Examples of use cases include: + +* Agriculture / weather modeling: number of rain events per year (Poisson), + amount of rainfall per event (Gamma), total rainfall per year (Tweedie / + Compound Poisson Gamma). +* Risk modeling / insurance policy pricing: number of claim events / + policyholder per year (Poisson), cost per event (Gamma), total cost per + policyholder per year (Tweedie / Compound Poisson Gamma). +* Predictive maintenance: number of production interruption events per year: + Poisson, duration of interruption: Gamma, total interruption time per year + (Tweedie / Compound Poisson Gamma). + + +.. topic:: References: + + .. [10] McCullagh, Peter; Nelder, John (1989). Generalized Linear Models, + Second Edition. Boca Raton: Chapman and Hall/CRC. ISBN 0-412-31760-5. + + .. [11] Jørgensen, B. (1992). The theory of exponential dispersion models + and analysis of deviance. Monografias de matemática, no. 51. See also + `Exponential dispersion model. + `_ + +Usage +----- + +:class:`TweedieRegressor` implements a generalized linear model for the +Tweedie distribution, that allows to model any of the above mentioned +distributions using the appropriate ``power`` parameter. In particular: + +- ``power = 0``: Normal distribution. Specific estimators such as + :class:`Ridge`, :class:`ElasticNet` are generally more appropriate in + this case. +- ``power = 1``: Poisson distribution. :class:`PoissonRegressor` is exposed + for convenience. However, it is strictly equivalent to + `TweedieRegressor(power=1, link='log')`. +- ``power = 2``: Gamma distribution. :class:`GammaRegressor` is exposed for + convenience. However, it is strictly equivalent to + `TweedieRegressor(power=2, link='log')`. +- ``power = 3``: Inverse Gaussian distribution. + +The link function is determined by the `link` parameter. + +Usage example:: + + >>> from sklearn.linear_model import TweedieRegressor + >>> reg = TweedieRegressor(power=1, alpha=0.5, link='log') + >>> reg.fit([[0, 0], [0, 1], [2, 2]], [0, 1, 2]) + TweedieRegressor(alpha=0.5, link='log', power=1) + >>> reg.coef_ + array([0.2463..., 0.4337...]) + >>> reg.intercept_ + -0.7638... + + +.. topic:: Examples: + + * :ref:`sphx_glr_auto_examples_linear_model_plot_poisson_regression_non_normal_loss.py` + * :ref:`sphx_glr_auto_examples_linear_model_plot_tweedie_regression_insurance_claims.py` + +Practical considerations +------------------------ + +The feature matrix `X` should be standardized before fitting. This ensures +that the penalty treats features equally. + +Since the linear predictor :math:`Xw` can be negative and Poisson, +Gamma and Inverse Gaussian distributions don't support negative values, it +is necessary to apply an inverse link function that guarantees the +non-negativeness. For example with `link='log'`, the inverse link function +becomes :math:`h(Xw)=\exp(Xw)`. + +If you want to model a relative frequency, i.e. counts per exposure (time, +volume, ...) you can do so by using a Poisson distribution and passing +:math:`y=\frac{\mathrm{counts}}{\mathrm{exposure}}` as target values +together with :math:`\mathrm{exposure}` as sample weights. For a concrete +example see e.g. +:ref:`sphx_glr_auto_examples_linear_model_plot_tweedie_regression_insurance_claims.py`. + +When performing cross-validation for the `power` parameter of +`TweedieRegressor`, it is advisable to specify an explicit `scoring` function, +because the default scorer :meth:`TweedieRegressor.score` is a function of +`power` itself. Stochastic Gradient Descent - SGD ================================= diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index 3f5999346401a..e1b7ae34f1647 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -459,7 +459,11 @@ In the binary case, balanced accuracy is equal to the arithmetic mean of (true positive rate) and `specificity `_ (true negative rate), or the area under the ROC curve with binary predictions rather than -scores. +scores: + +.. math:: + + \texttt{balanced-accuracy} = \frac{1}{2}\left( \frac{TP}{TP + FN} + \frac{TN}{TN + FP}\right ) If the classifier performs equally well on either class, this term reduces to the conventional accuracy (i.e., the number of correct predictions divided by @@ -555,11 +559,10 @@ Confusion matrix ---------------- The :func:`confusion_matrix` function evaluates -classification accuracy by computing the confusion matrix -with each row corresponding to the true class -`_. -(Wikipedia and other references may use different convention for axes.) - +classification accuracy by computing the `confusion matrix +`_ with each row corresponding +to the true class (Wikipedia and other references may use different convention +for axes). By definition, entry :math:`i, j` in a confusion matrix is the number of observations actually in group :math:`i`, but @@ -573,14 +576,27 @@ predicted to be in group :math:`j`. Here is an example:: [0, 0, 1], [1, 0, 2]]) -Here is a visual representation of such a confusion matrix (this figure comes -from the :ref:`sphx_glr_auto_examples_model_selection_plot_confusion_matrix.py` example): +:func:`plot_confusion_matrix` can be used to visually represent a confusion +matrix as shown in the +:ref:`sphx_glr_auto_examples_model_selection_plot_confusion_matrix.py` +example, which creates the following figure: .. image:: ../auto_examples/model_selection/images/sphx_glr_plot_confusion_matrix_001.png :target: ../auto_examples/model_selection/plot_confusion_matrix.html :scale: 75 :align: center +The parameter ``normalize`` allows to report ratios instead of counts. The +confusion matrix can be normalized in 3 different ways: ``'pred'``, ``'true'``, +and ``'all'`` which will divide the counts by the sum of each columns, rows, or +the entire matrix, respectively. + + >>> y_true = [0, 0, 0, 1, 1, 1, 1, 1] + >>> y_pred = [0, 1, 0, 1, 0, 1, 0, 1] + >>> confusion_matrix(y_true, y_pred, normalize='all') + array([[0.25 , 0.125], + [0.25 , 0.375]]) + For binary problems, we can get counts of true negatives, false positives, false negatives and true positives as follows:: @@ -1335,8 +1351,8 @@ the one-vs-rest algorithm computes the average of the ROC AUC scores for each class against all other classes. In both cases, the predicted labels are provided in an array with values from 0 to ``n_classes``, and the scores correspond to the probability estimates that a sample belongs to a particular -class. The OvO and OvR algorithms supports weighting uniformly -(``average='macro'``) and weighting by the prevalence (``average='weighted'``). +class. The OvO and OvR algorithms support weighting uniformly +(``average='macro'``) and by prevalence (``average='weighted'``). **One-vs-one Algorithm**: Computes the average AUC of all possible pairwise combinations of classes. [HT2001]_ defines a multiclass AUC metric weighted @@ -1367,10 +1383,10 @@ the keyword argument ``multiclass`` to ``'ovo'`` and ``average`` to ``'weighted'``. The ``'weighted'`` option returns a prevalence-weighted average as described in [FC2009]_. -**One-vs-rest Algorithm**: Computes the AUC of each class against the rest. -The algorithm is functionally the same as the multilabel case. To enable this -algorithm set the keyword argument ``multiclass`` to ``'ovr'``. Similar to -OvO, OvR supports two types of averaging: ``'macro'`` [F2006]_ and +**One-vs-rest Algorithm**: Computes the AUC of each class against the rest +[PD2000]_. The algorithm is functionally the same as the multilabel case. To +enable this algorithm set the keyword argument ``multiclass`` to ``'ovr'``. +Like OvO, OvR supports two types of averaging: ``'macro'`` [F2006]_ and ``'weighted'`` [F2001]_. In applications where a high false positive rate is not tolerable the parameter @@ -1409,6 +1425,10 @@ to the given limit. `_ Pattern Recognition Letters. 30. 27-38. + .. [PD2000] Provost, F., Domingos, P. (2000). Well-trained PETs: Improving + probability estimation trees (Section 6.2), CeDER Working Paper #IS-00-04, + Stern School of Business, New York University. + .. [F2006] Fawcett, T., 2006. `An introduction to ROC analysis. `_ Pattern Recognition Letters, 27(8), pp. 861-874. @@ -1682,7 +1702,7 @@ Discounted Cumulative Gain (DCG) and Normalized Discounted Cumulative Gain (NDCG) are ranking metrics; they compare a predicted order to ground-truth scores, such as the relevance of answers to a query. -from the Wikipedia page for Discounted Cumulative Gain: +From the Wikipedia page for Discounted Cumulative Gain: "Discounted cumulative gain (DCG) is a measure of ranking quality. In information retrieval, it is often used to measure effectiveness of web search @@ -1707,7 +1727,7 @@ relevant), NDCG can be used. For one sample, given the vector of continuous ground-truth values for each target :math:`y \in \mathbb{R}^{M}`, where :math:`M` is the number of outputs, and -the prediction :math:`\hat{y}`, which induces the ranking funtion :math:`f`, the +the prediction :math:`\hat{y}`, which induces the ranking function :math:`f`, the DCG score is .. math:: @@ -1718,8 +1738,8 @@ and the NDCG score is the DCG score divided by the DCG score obtained for .. topic:: References: - * Wikipedia entry for Discounted Cumulative Gain: - https://en.wikipedia.org/wiki/Discounted_cumulative_gain + * `Wikipedia entry for Discounted Cumulative Gain + `_ * Jarvelin, K., & Kekalainen, J. (2002). Cumulated gain-based evaluation of IR techniques. ACM Transactions on diff --git a/doc/modules/multiclass.rst b/doc/modules/multiclass.rst index 195ecc0adcf6f..606b4246a0b88 100644 --- a/doc/modules/multiclass.rst +++ b/doc/modules/multiclass.rst @@ -14,45 +14,138 @@ Multiclass and multilabel algorithms The :mod:`sklearn.multiclass` module implements *meta-estimators* to solve ``multiclass`` and ``multilabel`` classification problems -by decomposing such problems into binary classification problems. Multitarget +by decomposing such problems into binary classification problems. ``multioutput`` regression is also supported. -- **Multiclass classification** means a classification task with more than - two classes; e.g., classify a set of images of fruits which may be oranges, - apples, or pears. Multiclass classification makes the assumption that each - sample is assigned to one and only one label: a fruit can be either an - apple or a pear but not both at the same time. - -- **Multilabel classification** assigns to each sample a set of target - labels. This can be thought as predicting properties of a data-point - that are not mutually exclusive, such as topics that are relevant for a - document. A text might be about any of religion, politics, finance or - education at the same time or none of these. - -- **Multioutput regression** assigns each sample a set of target - values. This can be thought of as predicting several properties - for each data-point, such as wind direction and magnitude at a - certain location. - -- **Multioutput-multiclass classification** and **multi-task classification** - means that a single estimator has to handle several joint classification - tasks. This is both a generalization of the multi-label classification - task, which only considers binary classification, as well as a - generalization of the multi-class classification task. *The output format - is a 2d numpy array or sparse matrix.* - - The set of labels can be different for each output variable. - For instance, a sample could be assigned "pear" for an output variable that - takes possible values in a finite set of species such as "pear", "apple"; - and "blue" or "green" for a second output variable that takes possible values - in a finite set of colors such as "green", "red", "blue", "yellow"... - - This means that any classifiers handling multi-output - multiclass or multi-task classification tasks, - support the multi-label classification task as a special case. - Multi-task classification is similar to the multi-output - classification task with different model formulations. For - more information, see the relevant estimator documentation. +- **Multiclass classification**: classification task with more than two classes. + Each sample can only be labelled as one class. + + For example, classification using features extracted from a set of images of + fruit, where each image may either be of an orange, an apple, or a pear. + Each image is one sample and is labelled as one of the 3 possible classes. + Multiclass classification makes the assumption that each sample is assigned + to one and only one label - one sample cannot, for example, be both a pear + and an apple. + + Valid :term:`multiclass` representations for + :func:`~utils.multiclass.type_of_target` (`y`) are: + + - 1d or column vector containing more than two discrete values. An + example of a vector ``y`` for 3 samples: + + >>> import numpy as np + >>> y = np.array(['apple', 'pear', 'apple']) + >>> print(y) + ['apple' 'pear' 'apple'] + + - sparse :term:`binary` matrix of shape ``(n_samples, n_classes)`` with a + single element per row, where each column represents one class. An + example of a sparse :term:`binary` matrix ``y`` for 3 samples, where + the columns, in order, are orange, apple and pear: + + >>> from scipy import sparse + >>> row_ind = np.array([0, 1, 2]) + >>> col_ind = np.array([1, 2, 1]) + >>> y_sparse = sparse.csr_matrix((np.ones(3), (row_ind, col_ind))) + >>> print(y_sparse) + (0, 1) 1.0 + (1, 2) 1.0 + (2, 1) 1.0 + + +- **Multilabel classification**: classification task labelling each sample with + ``x`` labels from ``n_classes`` possible classes, where ``x`` can be 0 to + ``n_classes`` inclusive. This can be thought of as predicting properties of a + sample that are not mutually exclusive. Formally, a binary output is assigned + to each class, for every sample. Positive classes are indicated with 1 and + negative classes with 0 or -1. It is thus comparable to running ``n_classes`` + binary classification tasks, for example with + :class:`sklearn.multioutput.MultiOutputClassifier`. This approach treats + each label independently whereas multilabel classifiers *may* treat the + multiple classes simultaneously, accounting for correlated behavior among + them. + + For example, prediction of the topics relevant to a text document or video. + The document or video may be about one of 'religion', 'politics', 'finance' + or 'education', several of the topic classes or all of the topic classes. + + A valid representation of :term:`multilabel` `y` is an either dense or sparse + :term:`binary` matrix of shape ``(n_samples, n_classes)``. Each column + represents a class. The ``1``'s in each row denote the positive classes a + sample has been labelled with. An example of a dense matrix ``y`` for 3 + samples: + + >>> y = np.array([[1, 0, 0, 1], [0, 0, 1, 1], [0, 0, 0, 0]]) + >>> print(y) + [[1 0 0 1] + [0 0 1 1] + [0 0 0 0]] + + An example of the same ``y`` in sparse matrix form: + + >>> y_sparse = sparse.csr_matrix(y) + >>> print(y_sparse) + (0, 0) 1 + (0, 3) 1 + (1, 2) 1 + (1, 3) 1 + + +- **Multioutput regression**: predicts multiple numerical properties for each + sample. Each property is a numerical variable and the number of properties + to be predicted for each sample is greater than or equal to 2. Some estimators + that support multioutput regression are faster than just running ``n_output`` + estimators. + + For example, prediction of both wind speed and wind direction, in degrees, + using data obtained at a certain location. Each sample would be data + obtained at one location and both wind speed and direction would be + output for each sample. + + A valid representation of :term:`multioutput` `y` is a dense matrix of shape + ``(n_samples, n_classes)`` of floats. A column wise concatenation of + :term:`continuous` variables. An example of ``y`` for 3 samples: + + >>> y = np.array([[31.4, 94], [40.5, 109], [25.0, 30]]) + >>> print(y) + [[ 31.4 94. ] + [ 40.5 109. ] + [ 25. 30. ]] + + +- **Multioutput-multiclass classification** + (also known as **multitask classification**): + classification task which labels each sample with a set of **non-binary** + properties. Both the number of properties and the number of + classes per property is greater than 2. A single estimator thus + handles several joint classification tasks. This is both a generalization of + the multi\ *label* classification task, which only considers binary + attributes, as well as a generalization of the multi\ *class* classification + task, where only one property is considered. + + For example, classification of the properties "type of fruit" and "colour" + for a set of images of fruit. The property "type of fruit" has the possible + classes: "apple", "pear" and "orange". The property "colour" has the + possible classes: "green", "red", "yellow" and "orange". Each sample is an + image of a fruit, a label is output for both properties and each label is + one of the possible classes of the corresponding property. + + A valid representation of :term:`multioutput` `y` is a dense matrix of shape + ``(n_samples, n_classes)`` of class labels. A column wise concatenation of 1d + :term:`multiclass` variables. An example of ``y`` for 3 samples: + + >>> y = np.array([['apple', 'green'], ['orange', 'orange'], ['pear', 'green']]) + >>> print(y) + [['apple' 'green'] + ['orange' 'orange'] + ['pear' 'green']] + + Note that all classifiers handling multioutput-multiclass (also known as + multitask classification) tasks, support the multilabel classification task + as a special case. Multitask classification is similar to the multioutput + classification task with different model formulations. For more information, + see the relevant estimator documentation. + All scikit-learn classifiers are capable of multiclass classification, but the meta-estimators offered by :mod:`sklearn.multiclass` @@ -60,6 +153,26 @@ permit changing the way they handle more than two classes because this may have an effect on classifier performance (either in terms of generalization error or required computational resources). +**Summary** + ++-----------------+-------------+-------------+------------------------------------------+ +| | Number of | Target | Valid | +| | targets | cardinality | :func:`~utils.multiclass.type_of_target` | ++=================+=============+=============+==========================================+ +| Multiclass | 1 | >2 | - 'multiclass' | +| classification | | | | ++-----------------+-------------+-------------+------------------------------------------+ +| Multilabel | >1 | 2 (0 or 1) | - 'multilabel-indicator' | +| classification | | | | ++-----------------+-------------+-------------+------------------------------------------+ +| Multioutput | >1 | Continuous | - 'continuous-multioutput' | +| regression | | | | ++-----------------+-------------+-------------+------------------------------------------+ +| Multioutput- | >1 | >2 | - 'multiclass-multioutput' | +| multiclass | | | | +| classification | | | | ++-----------------+-------------+-------------+------------------------------------------+ + Below is a summary of the classifiers supported by scikit-learn grouped by strategy; you don't need the meta-estimators in this class if you're using one of these, unless you want custom multiclass behavior: @@ -94,7 +207,7 @@ if you're using one of these, unless you want custom multiclass behavior: - :class:`sklearn.gaussian_process.GaussianProcessClassifier` (setting multi_class = "one_vs_one") -- **Multiclass as One-Vs-All:** +- **Multiclass as One-Vs-The-Rest:** - :class:`sklearn.ensemble.GradientBoostingClassifier` - :class:`sklearn.gaussian_process.GaussianProcessClassifier` (setting multi_class = "one_vs_rest") @@ -167,7 +280,7 @@ This strategy, also known as **one-vs-all**, is implemented in per class. For each classifier, the class is fitted against all the other classes. In addition to its computational efficiency (only `n_classes` classifiers are needed), one advantage of this approach is its -interpretability. Since each class is represented by one and only one classifier, +interpretability. Since each class is represented by one and only one classifier, it is possible to gain knowledge about the class by inspecting its corresponding classifier. This is the most commonly used strategy and is a fair default choice. @@ -431,7 +544,7 @@ averaged together. Regressor Chain ================ -Regressor chains (see :class:`RegressorChain`) is analogous to -ClassifierChain as a way of combining a number of regressions -into a single multi-target model that is capable of exploiting +Regressor chains (see :class:`RegressorChain`) is analogous to +ClassifierChain as a way of combining a number of regressions +into a single multi-target model that is capable of exploiting correlations among targets. diff --git a/doc/modules/neighbors.rst b/doc/modules/neighbors.rst index 7f72aa68c38db..9aa27a53501b8 100644 --- a/doc/modules/neighbors.rst +++ b/doc/modules/neighbors.rst @@ -581,7 +581,7 @@ implementation with special data types. The precomputed neighbors training point as its own neighbor in the count of `n_neighbors`. However, for compatibility reasons with other estimators which use the other definition, one extra neighbor will be computed when `mode == 'distance'`. - To maximise compatiblity with all estimators, a safe choice is to always + To maximise compatibility with all estimators, a safe choice is to always include one extra neighbor in a custom nearest neighbors estimator, since unnecessary neighbors will be filtered by following estimators. diff --git a/doc/modules/partial_dependence.rst b/doc/modules/partial_dependence.rst index a538f1156b748..612dbcefce01d 100644 --- a/doc/modules/partial_dependence.rst +++ b/doc/modules/partial_dependence.rst @@ -92,22 +92,76 @@ generated. The ``values`` field returned by used in the grid for each target feature. They also correspond to the axis of the plots. -For each value of the 'target' features in the ``grid`` the partial -dependence function needs to marginalize the predictions of the estimator -over all possible values of the 'complement' features. With the ``'brute'`` -method, this is done by replacing every target feature value of ``X`` by those -in the grid, and computing the average prediction. - -In decision trees this can be evaluated efficiently without reference to the -training data (``'recursion'`` method). For each grid point a weighted tree -traversal is performed: if a split node involves a 'target' feature, the -corresponding left or right branch is followed, otherwise both branches are -followed, each branch is weighted by the fraction of training samples that -entered that branch. Finally, the partial dependence is given by a weighted -average of all visited leaves. Note that with the ``'recursion'`` method, -``X`` is only used to generate the grid, not to compute the averaged -predictions. The averaged predictions will always be computed on the data with -which the trees were trained. +Mathematical Definition +^^^^^^^^^^^^^^^^^^^^^^^ + +Let :math:`X_S` be the set of target features (i.e. the `features` parameter) +and let :math:`X_C` be its complement. + +The partial dependence of the response :math:`f` at a point :math:`x_S` is +defined as: + +.. math:: + + pd_{X_S}(x_S) &\overset{def}{=} \mathbb{E}_{X_C}\left[ f(x_S, X_C) \right]\\ + &= \int f(x_S, x_C) p(x_C) dx_C, + +where :math:`f(x_S, x_C)` is the response function (:term:`predict`, +:term:`predict_proba` or :term:`decision_function`) for a given sample whose +values are defined by :math:`x_S` for the features in :math:`X_S`, and by +:math:`x_C` for the features in :math:`X_C`. Note that :math:`x_S` and +:math:`x_C` may be tuples. + +Computing this integral for various values of :math:`x_S` produces a plot as +above. + +Computation methods +^^^^^^^^^^^^^^^^^^^ + +There are two main methods to approximate the integral above, namely the +'brute' and 'recursion' methods. The `method` parameter controls which method +to use. + +The 'brute' method is a generic method that works with any estimator. It +approximates the above integral by computing an average over the data `X`: + +.. math:: + + pd_{X_S}(x_S) \approx \frac{1}{n_\text{samples}} \sum_{i=1}^n f(x_S, x_C^{(i)}), + +where :math:`x_C^{(i)}` is the value of the i-th sample for the features in +:math:`X_C`. For each value of :math:`x_S`, this method requires a full pass +over the dataset `X` which is computationally intensive. + +The 'recursion' method is faster than the 'brute' method, but it is only +supported by some tree-based estimators. It is computed as follows. For a +given point :math:`x_S`, a weighted tree traversal is performed: if a split +node involves a 'target' feature, the corresponding left or right branch is +followed; otherwise both branches are followed, each branch being weighted +by the fraction of training samples that entered that branch. Finally, the +partial dependence is given by a weighted average of all the visited leaves +values. + +With the 'brute' method, the parameter `X` is used both for generating the +grid of values :math:`x_S` and the complement feature values :math:`x_C`. +However with the 'recursion' method, `X` is only used for the grid values: +implicitly, the :math:`x_C` values are those of the training data. + +By default, the 'recursion' method is used on tree-based estimators that +support it, and 'brute' is used for the rest. + +.. _pdp_method_differences: + +.. note:: + + While both methods should be close in general, they might differ in some + specific settings. The 'brute' method assumes the existence of the + data points :math:`(x_S, x_C^{(i)})`. When the features are correlated, + such artificial samples may have a very low probability mass. The 'brute' + and 'recursion' methods will likely disagree regarding the value of the + partial dependence, because they will treat these unlikely + samples differently. Remember, however, that the primary assumption for + interpreting PDPs is that the features should be independent. .. rubric:: Footnotes diff --git a/doc/modules/permutation_importance.rst b/doc/modules/permutation_importance.rst index 1ea03ffc11ddb..aa28aba6827da 100644 --- a/doc/modules/permutation_importance.rst +++ b/doc/modules/permutation_importance.rst @@ -7,7 +7,7 @@ Permutation feature importance .. currentmodule:: sklearn.inspection Permutation feature importance is a model inspection technique that can be used -for any :term:`fitted` :term:`estimator` when the data is rectangular. This is +for any :term:`fitted` :term:`estimator` when the data is tabular. This is especially useful for non-linear or opaque :term:`estimators`. The permutation feature importance is defined to be the decrease in a model score when a single feature value is randomly shuffled [1]_. This procedure breaks the relationship @@ -19,43 +19,118 @@ different permutations of the feature. The :func:`permutation_importance` function calculates the feature importance of :term:`estimators` for a given dataset. The ``n_repeats`` parameter sets the number of times a feature is randomly shuffled and returns a sample of feature -importances. Permutation importances can either be computed on the training set -or an held-out testing or validation set. Using a held-out set makes it -possible to highlight which features contribute the most to the generalization -power of the inspected model. Features that are important on the training set -but not on the held-out set might cause the model to overfit. - -Note that features that are deemed non-important for some model with a -low predictive performance could be highly predictive for a model that -generalizes better. The conclusions should always be drawn in the context of -the specific model under inspection and cannot be automatically generalized to -the intrinsic predictive value of the features by them-selves. Therefore it is -always important to evaluate the predictive power of a model using a held-out -set (or better with cross-validation) prior to computing importances. +importances. + +Let's consider the following trained regression model:: + + >>> from sklearn.datasets import load_diabetes + >>> from sklearn.model_selection import train_test_split + >>> from sklearn.linear_model import Ridge + >>> diabetes = load_diabetes() + >>> X_train, X_val, y_train, y_val = train_test_split( + ... diabetes.data, diabetes.target, random_state=0) + ... + >>> model = Ridge(alpha=1e-2).fit(X_train, y_train) + >>> model.score(X_val, y_val) + 0.356... + +Its validation performance, measured via the :math:`R^2` score, is +significantly larger than the chance level. This makes it possible to use the +:func:`permutation_importance` function to probe which features are most +predictive:: + + >>> from sklearn.inspection import permutation_importance + >>> r = permutation_importance(model, X_val, y_val, + ... n_repeats=30, + ... random_state=0) + ... + >>> for i in r.importances_mean.argsort()[::-1]: + ... if r.importances_mean[i] - 2 * r.importances_std[i] > 0: + ... print(f"{diabetes.feature_names[i]:<8}" + ... f"{r.importances_mean[i]:.3f}" + ... f" +/- {r.importances_std[i]:.3f}") + ... + s5 0.204 +/- 0.050 + bmi 0.176 +/- 0.048 + bp 0.088 +/- 0.033 + sex 0.056 +/- 0.023 + +Note that the importance values for the top features represent a large +fraction of the reference score of 0.356. + +Permutation importances can be computed either on the training set or on a +held-out testing or validation set. Using a held-out set makes it possible to +highlight which features contribute the most to the generalization power of the +inspected model. Features that are important on the training set but not on the +held-out set might cause the model to overfit. + +.. warning:: + + Features that are deemed of **low importance for a bad model** (low + cross-validation score) could be **very important for a good model**. + Therefore it is always important to evaluate the predictive power of a model + using a held-out set (or better with cross-validation) prior to computing + importances. Permutation importance does not reflect to the intrinsic + predictive value of a feature by itself but **how important this feature is + for a particular model**. + +Outline of the permutation importance algorithm +----------------------------------------------- + +- Inputs: fitted predictive model :math:`m`, tabular dataset (training or + validation) :math:`D`. +- Compute the reference score :math:`s` of the model :math:`m` on data + :math:`D` (for instance the accuracy for a classifier or the :math:`R^2` for + a regressor). +- For each feature :math:`j` (column of :math:`D`): + + - For each repetition :math:`k` in :math:`{1, ..., K}`: + + - Randomly shuffle column :math:`j` of dataset :math:`D` to generate a + corrupted version of the data named :math:`\tilde{D}_{k,j}`. + - Compute the score :math:`s_{k,j}` of model :math:`m` on corrupted data + :math:`\tilde{D}_{k,j}`. + + - Compute importance :math:`i_j` for feature :math:`f_j` defined as: + + .. math:: i_j = s - \frac{1}{K} \sum_{k=1}^{K} s_{k,j} Relation to impurity-based importance in trees ---------------------------------------------- -Tree based models provides a different measure of feature importances based -on the mean decrease in impurity (MDI, the splitting criterion). This gives -importance to features that may not be predictive on unseen data. The -permutation feature importance avoids this issue, since it can be applied to -unseen data. Furthermore, impurity-based feature importance for trees -are strongly biased and favor high cardinality features -(typically numerical features). Permutation-based feature importances do not -exhibit such a bias. Additionally, the permutation feature importance may use -an arbitrary metric on the tree's predictions. These two methods of obtaining -feature importance are explored in: +Tree-based models provide an alternative measure of :ref:`feature importances +based on the mean decrease in impurity ` +(MDI). Impurity is quantified by the splitting criterion of the decision trees +(Gini, Entropy or Mean Squared Error). However, this method can give high +importance to features that may not be predictive on unseen data when the model +is overfitting. Permutation-based feature importance, on the other hand, avoids +this issue, since it can be computed on unseen data. + +Furthermore, impurity-based feature importance for trees are **strongly +biased** and **favor high cardinality features** (typically numerical features) +over low cardinality features such as binary features or categorical variables +with a small number of possible categories. + +Permutation-based feature importances do not exhibit such a bias. Additionally, +the permutation feature importance may be computed performance metric on the +model predictions predictions and can be used to analyze any model class (not +just tree-based models). + +The following example highlights the limitations of impurity-based feature +importance in contrast to permutation-based feature importance: :ref:`sphx_glr_auto_examples_inspection_plot_permutation_importance.py`. -Strongly correlated features ----------------------------- +Misleading values on strongly correlated features +------------------------------------------------- When two features are correlated and one of the features is permuted, the model -will still have access to the feature through its correlated feature. This will -result in a lower importance for both features, where they might *actually* be -important. One way to handle this is to cluster features that are correlated -and only keep one feature from each cluster. This use case is explored in: +will still have access to the feature through its correlated feature. This will +result in a lower importance value for both features, where they might +*actually* be important. + +One way to handle this is to cluster features that are correlated and only +keep one feature from each cluster. This strategy is explored in the following +example: :ref:`sphx_glr_auto_examples_inspection_plot_permutation_importance_multicollinear.py`. .. topic:: Examples: diff --git a/doc/modules/preprocessing.rst b/doc/modules/preprocessing.rst index 3e41c592fbbdc..fcf3d623a452f 100644 --- a/doc/modules/preprocessing.rst +++ b/doc/modules/preprocessing.rst @@ -559,11 +559,12 @@ parameter allows the user to specify a category for each feature to be dropped. This is useful to avoid co-linearity in the input matrix in some classifiers. Such functionality is useful, for example, when using non-regularized regression (:class:`LinearRegression `), -since co-linearity would cause the covariance matrix to be non-invertible. -When this paramenter is not None, ``handle_unknown`` must be set to +since co-linearity would cause the covariance matrix to be non-invertible. +When this parameter is not None, ``handle_unknown`` must be set to ``error``:: - >>> X = [['male', 'from US', 'uses Safari'], ['female', 'from Europe', 'uses Firefox']] + >>> X = [['male', 'from US', 'uses Safari'], + ... ['female', 'from Europe', 'uses Firefox']] >>> drop_enc = preprocessing.OneHotEncoder(drop='first').fit(X) >>> drop_enc.categories_ [array(['female', 'male'], dtype=object), array(['from Europe', 'from US'], dtype=object), array(['uses Firefox', 'uses Safari'], dtype=object)] @@ -571,8 +572,26 @@ When this paramenter is not None, ``handle_unknown`` must be set to array([[1., 1., 1.], [0., 0., 0.]]) -See :ref:`dict_feature_extraction` for categorical features that are represented -as a dict, not as scalars. +One might want to drop one of the two columns only for features with 2 +categories. In this case, you can set the parameter `drop='if_binary'`. + + >>> X = [['male', 'US', 'Safari'], + ... ['female', 'Europe', 'Firefox'], + ... ['female', 'Asia', 'Chrome']] + >>> drop_enc = preprocessing.OneHotEncoder(drop='if_binary').fit(X) + >>> drop_enc.categories_ + [array(['female', 'male'], dtype=object), array(['Asia', 'Europe', 'US'], dtype=object), array(['Chrome', 'Firefox', 'Safari'], dtype=object)] + >>> drop_enc.transform(X).toarray() + array([[1., 0., 0., 1., 0., 0., 1.], + [0., 0., 1., 0., 0., 1., 0.], + [0., 1., 0., 0., 1., 0., 0.]]) + +In the transformed `X`, the first column is the encoding of the feature with +categories "male"/"female", while the remaining 6 columns is the encoding of +the 2 features with respectively 3 categories each. + +See :ref:`dict_feature_extraction` for categorical features that are +represented as a dict, not as scalars. .. _preprocessing_discretization: diff --git a/doc/modules/svm.rst b/doc/modules/svm.rst index 03020cfd2252c..706a9ff559aa8 100644 --- a/doc/modules/svm.rst +++ b/doc/modules/svm.rst @@ -267,10 +267,11 @@ that sets the parameter ``C`` of class ``class_label`` to ``C * value``. :scale: 75 -:class:`SVC`, :class:`NuSVC`, :class:`SVR`, :class:`NuSVR` and -:class:`OneClassSVM` implement also weights for individual samples in method -``fit`` through keyword ``sample_weight``. Similar to ``class_weight``, these -set the parameter ``C`` for the i-th example to ``C * sample_weight[i]``. +:class:`SVC`, :class:`NuSVC`, :class:`SVR`, :class:`NuSVR`, :class:`LinearSVC`, +:class:`LinearSVR` and :class:`OneClassSVM` implement also weights for +individual samples in method ``fit`` through keyword ``sample_weight``. Similar +to ``class_weight``, these set the parameter ``C`` for the i-th example to +``C * sample_weight[i]``. .. figure:: ../auto_examples/svm/images/sphx_glr_plot_weighted_samples_001.png @@ -319,10 +320,10 @@ floating point values instead of integer values:: >>> from sklearn import svm >>> X = [[0, 0], [2, 2]] >>> y = [0.5, 2.5] - >>> clf = svm.SVR() - >>> clf.fit(X, y) + >>> regr = svm.SVR() + >>> regr.fit(X, y) SVR() - >>> clf.predict([[1, 1]]) + >>> regr.predict([[1, 1]]) array([1.5]) @@ -455,13 +456,13 @@ The *kernel function* can be any of the following: * linear: :math:`\langle x, x'\rangle`. - * polynomial: :math:`(\gamma \langle x, x'\rangle + r)^d`. + * polynomial: :math:`(\gamma \langle x, x'\rangle + r)^d`, where :math:`d` is specified by keyword ``degree``, :math:`r` by ``coef0``. - * rbf: :math:`\exp(-\gamma \|x-x'\|^2)`. :math:`\gamma` is + * rbf: :math:`\exp(-\gamma \|x-x'\|^2)`, where :math:`\gamma` is specified by keyword ``gamma``, must be greater than 0. - * sigmoid (:math:`\tanh(\gamma \langle x,x'\rangle + r)`), + * sigmoid :math:`\tanh(\gamma \langle x,x'\rangle + r)`, where :math:`r` is specified by ``coef0``. Different kernels are specified by keyword kernel at initialization:: diff --git a/doc/support.rst b/doc/support.rst index 5dd52c01030f0..75f9f4cee9a6d 100644 --- a/doc/support.rst +++ b/doc/support.rst @@ -73,16 +73,13 @@ Note: gists are git cloneable repositories and thus you can use git to push datafiles to them. -.. _irc: +.. _gitter: -IRC +Gitter === -Some developers like to hang out on channel ``#scikit-learn`` on -``irc.freenode.net``. - -If you do not have an IRC client or are behind a firewall this web -client works fine: https://webchat.freenode.net +Some developers like to hang out on scikit-learn Gitter room: +https://gitter.im/scikit-learn/scikit-learn. .. _documentation_resources: diff --git a/doc/templates/index.html b/doc/templates/index.html index 9a60b36ddeae2..e17111fb48eef 100644 --- a/doc/templates/index.html +++ b/doc/templates/index.html @@ -8,7 +8,7 @@

    scikit-learn

    Machine Learning in Python

    Getting Started - Whats New in {{ version }} + What's New in {{ release }} GitHub
    @@ -115,7 +115,7 @@

    Machine Learning in grid search, cross validation, metrics, - and more...

    + and more...

  • On-going development: What's new (Changelog)
  • +
  • Scikit-learn from 0.23 requires Python 3.6 or greater. +
  • +
  • March 2020. scikit-learn 0.22.2 is available for download (Changelog). +
  • January 2020. scikit-learn 0.22.1 is available for download (Changelog). +
  • December 2019. scikit-learn 0.22 is available for download (Changelog). +
  • Scikit-learn from 0.21 requires Python 3.5 or greater.
  • July 2019. scikit-learn 0.21.3 (Changelog) and 0.20.4 (Changelog) are available for download. @@ -180,6 +186,7 @@

    Community

  • Questions? See FAQ and stackoverflow
  • Mailing list: scikit-learn@python.org
  • Gitter: gitter.im/scikit-learn
  • +
  • Communication on all channels should respect PSF's code of conduct.
  • diff --git a/doc/themes/scikit-learn-modern/javascript.html b/doc/themes/scikit-learn-modern/javascript.html index bdeab8abb9f42..fc0dca1040e03 100644 --- a/doc/themes/scikit-learn-modern/javascript.html +++ b/doc/themes/scikit-learn-modern/javascript.html @@ -114,7 +114,7 @@ prevScrollpos = lastScrollTop; }; - /*** high preformance scroll event listener***/ + /*** high performance scroll event listener***/ var raf = window.requestAnimationFrame || window.webkitRequestAnimationFrame || window.mozRequestAnimationFrame || diff --git a/doc/themes/scikit-learn-modern/layout.html b/doc/themes/scikit-learn-modern/layout.html index f32c6f94d47e5..6f29cf52f7c91 100644 --- a/doc/themes/scikit-learn-modern/layout.html +++ b/doc/themes/scikit-learn-modern/layout.html @@ -64,7 +64,7 @@ Prev {%- endif %} {%- if parents -%} - Up + Up {%- else %} Up {%- endif %} @@ -77,7 +77,7 @@ {%- if pagename != "install" %} diff --git a/doc/themes/scikit-learn-modern/static/css/theme.css b/doc/themes/scikit-learn-modern/static/css/theme.css index c6738bb760d44..a77fb03e36f65 100644 --- a/doc/themes/scikit-learn-modern/static/css/theme.css +++ b/doc/themes/scikit-learn-modern/static/css/theme.css @@ -37,6 +37,7 @@ code { background-color: #ecf0f3; border-radius: 0.2rem; white-space: nowrap; + padding: 0.15rem; } nav { @@ -512,11 +513,21 @@ div.sk-sidebar-toc-logo { div.sk-sidebar-toc-wrapper { font-size: 0.9rem; - width: 120%; + width: 252px; overflow-x: hidden; overflow-y: scroll; height: 100vh; padding-right: 1.75rem; + + /* Hide scrollbar for IE and Edge */ + -ms-overflow-style: none; + + /* Hide scrollbar for Firefox */ + scrollbar-width: none; +} + +div.sk-sidebar-toc-wrapper::-webkit-scrollbar { + display: none; } div.sk-sidebar-toc-wrapper::after { @@ -823,6 +834,10 @@ div.body img { height: unset!important; /* Needed because sphinx sets the height */ } +div.body dd > p { + hyphens: none; +} + img.align-center, .figure.align-center, object.align-center { display: block; margin-left: auto; diff --git a/doc/themes/scikit-learn-modern/static/js/searchtools.js b/doc/themes/scikit-learn-modern/static/js/searchtools.js index ca53abe4f0038..0d4ca2328b079 100644 --- a/doc/themes/scikit-learn-modern/static/js/searchtools.js +++ b/doc/themes/scikit-learn-modern/static/js/searchtools.js @@ -11,7 +11,9 @@ * - Removes ajax call to get context for each result * - Adjusts Search.query to remove duplicates in search results. * - Adjusts Scorer to rank objects higher. - * - Adds Search._total_results to limit the number of search results. + * - Adds Search._total_non_object_results to limit the number of search non + * object results. Object results do not perform another GET resquest, so they + * are cheap to display. */ if (!Scorer) { @@ -63,10 +65,10 @@ var Search = { _index: null, _queued_query: null, _pulse_status: -1, - _total_results: 10, + _total_non_object_results: 10, htmlToText: function (htmlString) { - var htmlString = htmlString.replace(//g, ""); + var htmlString = htmlString.replace(//g, ""); var htmlElement = document.createElement("span"); htmlElement.innerHTML = htmlString; $(htmlElement) @@ -218,22 +220,23 @@ var Search = { objectterms.slice(i + 1, objectterms.length) ); - if (results.length < this._total_results) { - results = $u.uniq(results.concat( - this.performObjectSearch(objectterms[i], others) - ), false, function (item) {return item[1]}); - } + results = $u.uniq(results.concat( + this.performObjectSearch(objectterms[i], others) + ), false, function (item) {return item[1]}); } - if (results.length < this._total_results) { - // lookup as search terms in fulltext - results = results.concat( - this.performTermsSearch(searchterms, excluded, terms, titleterms) - ); - } + var total_object_results = results.length; + + // lookup as search terms in fulltext + results = results.concat( + this.performTermsSearch(searchterms, excluded, terms, titleterms) + ); - if (results.length > this._total_results) { - results = results.slice(0, this._total_results); + // Only have _total_non_object_results results above the number of + // total number of object results + var results_limit = total_object_results + this._total_non_object_results + if (results.length > results_limit) { + results = results.slice(0, results_limit); } // let the scorer override scores with a custom scoring function diff --git a/doc/tutorial/machine_learning_map/pyparsing.py b/doc/tutorial/machine_learning_map/pyparsing.py index 20690df7aec47..0c5fca5cf891d 100644 --- a/doc/tutorial/machine_learning_map/pyparsing.py +++ b/doc/tutorial/machine_learning_map/pyparsing.py @@ -1020,20 +1020,14 @@ def _trim_arity(func, maxargs=2): limit = [0] foundArity = [False] - # traceback return data structure changed in Py3.5 - normalize back to plain tuples - if system_version[:2] >= (3,5): - def extract_stack(limit=0): - # special handling for Python 3.5.0 - extra deep call stack by 1 - offset = -3 if system_version == (3,5,0) else -2 - frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset] - return [(frame_summary.filename, frame_summary.lineno)] - def extract_tb(tb, limit=0): - frames = traceback.extract_tb(tb, limit=limit) - frame_summary = frames[-1] - return [(frame_summary.filename, frame_summary.lineno)] - else: - extract_stack = traceback.extract_stack - extract_tb = traceback.extract_tb + def extract_stack(limit=0): + offset = -2 + frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset] + return [(frame_summary.filename, frame_summary.lineno)] + def extract_tb(tb, limit=0): + frames = traceback.extract_tb(tb, limit=limit) + frame_summary = frames[-1] + return [(frame_summary.filename, frame_summary.lineno)] # synthesize what would be returned by traceback.extract_stack at the call to # user's parse action 'func', so that we don't incur call penalty at parse time diff --git a/doc/tutorial/statistical_inference/unsupervised_learning.rst b/doc/tutorial/statistical_inference/unsupervised_learning.rst index f5aaac3a81236..b87fb64ec8d9b 100644 --- a/doc/tutorial/statistical_inference/unsupervised_learning.rst +++ b/doc/tutorial/statistical_inference/unsupervised_learning.rst @@ -21,14 +21,12 @@ K-means clustering ------------------- Note that there exist a lot of different clustering criteria and associated -algorithms. The simplest clustering algorithm is -:ref:`k_means`. +algorithms. The simplest clustering algorithm is :ref:`k_means`. .. image:: /auto_examples/cluster/images/sphx_glr_plot_cluster_iris_002.png - :target: ../../auto_examples/cluster/plot_cluster_iris.html - :scale: 70 - :align: right - + :target: ../../auto_examples/cluster/plot_cluster_iris.html + :scale: 70 + :align: center :: @@ -172,21 +170,40 @@ With agglomerative clustering, it is possible to specify which samples can be clustered together by giving a connectivity graph. Graphs in scikit-learn are represented by their adjacency matrix. Often, a sparse matrix is used. This can be useful, for instance, to retrieve connected regions (sometimes -also referred to as connected components) when -clustering an image: +also referred to as connected components) when clustering an image. .. image:: /auto_examples/cluster/images/sphx_glr_plot_coin_ward_segmentation_001.png - :target: ../../auto_examples/cluster/plot_coin_ward_segmentation.html - :scale: 40 - :align: right + :target: ../../auto_examples/cluster/plot_coin_ward_segmentation.html + :scale: 40 + :align: center + +:: -.. literalinclude:: ../../auto_examples/cluster/plot_coin_ward_segmentation.py - :lines: 21-45 + >>> from skimage.data import coins + >>> from scipy.ndimage.filters import gaussian_filter + >>> from skimage.transform import rescale + >>> rescaled_coins = rescale( + ... gaussian_filter(coins(), sigma=2), + ... 0.2, mode='reflect', anti_aliasing=False, multichannel=False + ... ) + >>> X = np.reshape(rescaled_coins, (-1, 1)) -.. - >>> from sklearn.feature_extraction.image import grid_to_graph - >>> connectivity = grid_to_graph(*face.shape) +We need a vectorized version of the image. `'rescaled_coins'` is a down-scaled +version of the coins image to speed up the process:: + + >>> from sklearn.feature_extraction import grid_to_graph + >>> connectivity = grid_to_graph(*rescaled_coins.shape) +Define the graph structure of the data. Pixels connected to their neighbors:: + + >>> n_clusters = 27 # number of regions + + >>> from sklearn.cluster import AgglomerativeClustering + >>> ward = AgglomerativeClustering(n_clusters=n_clusters, linkage='ward', + ... connectivity=connectivity) + >>> ward.fit(X) + AgglomerativeClustering(connectivity=..., n_clusters=27) + >>> label = np.reshape(ward.labels_, rescaled_coins.shape) Feature agglomeration ...................... @@ -199,9 +216,9 @@ clustering in the feature direction, in other words clustering the transposed data. .. image:: /auto_examples/cluster/images/sphx_glr_plot_digits_agglomeration_001.png - :target: ../../auto_examples/cluster/plot_digits_agglomeration.html - :align: right - :scale: 57 + :target: ../../auto_examples/cluster/plot_digits_agglomeration.html + :align: center + :scale: 57 :: diff --git a/doc/tutorial/text_analytics/skeletons/exercise_02_sentiment.py b/doc/tutorial/text_analytics/skeletons/exercise_02_sentiment.py index 11b1ff07acf7e..23299f5f01b3d 100644 --- a/doc/tutorial/text_analytics/skeletons/exercise_02_sentiment.py +++ b/doc/tutorial/text_analytics/skeletons/exercise_02_sentiment.py @@ -2,7 +2,7 @@ Sentiment analysis can be casted as a binary text classification problem, that is fitting a linear classifier on features extracted from the text -of the user messages so as to guess wether the opinion of the author is +of the user messages so as to guess whether the opinion of the author is positive or negative. In this examples we will use a movie review dataset. diff --git a/doc/tutorial/text_analytics/solutions/exercise_02_sentiment.py b/doc/tutorial/text_analytics/solutions/exercise_02_sentiment.py index 9f747694064ac..434bece341975 100644 --- a/doc/tutorial/text_analytics/solutions/exercise_02_sentiment.py +++ b/doc/tutorial/text_analytics/solutions/exercise_02_sentiment.py @@ -2,7 +2,7 @@ Sentiment analysis can be casted as a binary text classification problem, that is fitting a linear classifier on features extracted from the text -of the user messages so as to guess wether the opinion of the author is +of the user messages so as to guess whether the opinion of the author is positive or negative. In this examples we will use a movie review dataset. diff --git a/doc/visualizations.rst b/doc/visualizations.rst index 4b6f7ea34febb..47d826602b62f 100644 --- a/doc/visualizations.rst +++ b/doc/visualizations.rst @@ -72,6 +72,7 @@ Functions .. autosummary:: inspection.plot_partial_dependence + metrics.plot_confusion_matrix metrics.plot_precision_recall_curve metrics.plot_roc_curve @@ -84,5 +85,6 @@ Display Objects .. autosummary:: inspection.PartialDependenceDisplay + metrics.ConfusionMatrixDisplay metrics.PrecisionRecallDisplay metrics.RocCurveDisplay diff --git a/doc/whats_new.rst b/doc/whats_new.rst index a9097d765886e..7b84374bd5146 100644 --- a/doc/whats_new.rst +++ b/doc/whats_new.rst @@ -12,6 +12,7 @@ on libraries.io to be notified when new versions are released. .. toctree:: :maxdepth: 1 + Version 0.23 Version 0.22 Version 0.21 Version 0.20 diff --git a/doc/whats_new/_contributors.rst b/doc/whats_new/_contributors.rst index b148c7f1139ea..cc3957eca1592 100644 --- a/doc/whats_new/_contributors.rst +++ b/doc/whats_new/_contributors.rst @@ -175,3 +175,5 @@ .. _Thomas Fan: https://github.com/thomasjpfan .. _Nicolas Hug: https://github.com/NicolasHug + +.. _Guillaume Lemaitre: https://github.com/glemaitre diff --git a/doc/whats_new/v0.20.rst b/doc/whats_new/v0.20.rst index 4e3a4891b70e2..2eaf3199fbc3c 100644 --- a/doc/whats_new/v0.20.rst +++ b/doc/whats_new/v0.20.rst @@ -709,7 +709,7 @@ Support for Python 3.3 has been officially dropped. - |Feature| |Fix| :class:`decomposition.SparsePCA` now exposes ``normalize_components``. When set to True, the train and test data are - centered with the train mean repsectively during the fit phase and the + centered with the train mean respectively during the fit phase and the transform phase. This fixes the behavior of SparsePCA. When set to False, which is the default, the previous abnormal behaviour still holds. The False value is for backward compatibility and should not be used. :issue:`11585` diff --git a/doc/whats_new/v0.21.rst b/doc/whats_new/v0.21.rst index 59e3774e76c69..94099723dd0ec 100644 --- a/doc/whats_new/v0.21.rst +++ b/doc/whats_new/v0.21.rst @@ -295,7 +295,7 @@ Support for Python 3.4 and below has been officially dropped. ...................... - |MajorFeature| A new clustering algorithm: :class:`cluster.OPTICS`: an - algoritm related to :class:`cluster.DBSCAN`, that has hyperparameters easier + algorithm related to :class:`cluster.DBSCAN`, that has hyperparameters easier to set and that scales better, by :user:`Shane `, `Adrin Jalali`_, :user:`Erich Schubert `, `Hanmin Qin`_, and :user:`Assia Benbihi `. diff --git a/doc/whats_new/v0.22.rst b/doc/whats_new/v0.22.rst index 46aef104418ed..4f62f88be9071 100644 --- a/doc/whats_new/v0.22.rst +++ b/doc/whats_new/v0.22.rst @@ -2,12 +2,169 @@ .. currentmodule:: sklearn +.. _changes_0_22_2: + +Version 0.22.2.post1 +==================== + +**March 3 2020** + +The 0.22.2.post1 release includes a packaging fix for the source distribution +but the content of the packages is otherwise identical to the content of the +wheels with the 0.22.2 version (without the .post1 suffix). Both contain the +following changes. + +Changelog +--------- + +:mod:`sklearn.impute` +..................... + +- |Efficiency| Reduce :func:`impute.KNNImputer` asymptotic memory usage by + chunking pairwise distance computation. + :pr:`16397` by `Joel Nothman`_. + +:mod:`sklearn.metrics` +...................... + +- |Fix| Fixed a bug in :func:`metrics.plot_roc_curve` where + the name of the estimator was passed in the :class:`metrics.RocCurveDisplay` + instead of the parameter `name`. It results in a different plot when calling + :meth:`metrics.RocCurveDisplay.plot` for the subsequent times. + :pr:`16500` by :user:`Guillaume Lemaitre `. + +- |Fix| Fixed a bug in :func:`metrics.plot_precision_recall_curve` where the + name of the estimator was passed in the + :class:`metrics.PrecisionRecallDisplay` instead of the parameter `name`. It + results in a different plot when calling + :meth:`metrics.PrecisionRecallDisplay.plot` for the subsequent times. + :pr:`16505` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.neighbors` +.............................. + +- |Fix| Fix a bug which converted a list of arrays into a 2-D object + array instead of a 1-D array containing NumPy arrays. This bug + was affecting :meth:`neighbors.NearestNeighbors.radius_neighbors`. + :pr:`16076` by :user:`Guillaume Lemaitre ` and + :user:`Alex Shacked `. + +.. _changes_0_22_1: + +Version 0.22.1 +============== + +**January 2 2020** + +This is a bug-fix release to primarily resolve some packaging issues in version +0.22.0. It also includes minor documentation improvements and some bug fixes. + +Changelog +--------- + + +:mod:`sklearn.cluster` +...................... + +- |Fix| :class:`cluster.KMeans` with ``algorithm="elkan"`` now uses the same + stopping criterion as with the default ``algorithm="full"``. :pr:`15930` by + :user:`inder128`. + +:mod:`sklearn.inspection` +......................... + +- |Fix| :func:`inspection.permutation_importance` will return the same + `importances` when a `random_state` is given for both `n_jobs=1` or + `n_jobs>1` both with shared memory backends (thread-safety) and + isolated memory, process-based backends. + Also avoid casting the data as object dtype and avoid read-only error + on large dataframes with `n_jobs>1` as reported in :issue:`15810`. + Follow-up of :pr:`15898` by :user:`Shivam Gargsya `. + :pr:`15933` by :user:`Guillaume Lemaitre ` and `Olivier Grisel`_. + +- |Fix| :func:`inspection.plot_partial_dependence` and + :meth:`inspection.PartialDependenceDisplay.plot` now consistently checks + the number of axes passed in. :pr:`15760` by `Thomas Fan`_. + +:mod:`sklearn.metrics` +...................... + +- |Fix| :func:`metrics.plot_confusion_matrix` now raises error when `normalize` + is invalid. Previously, it runs fine with no normalization. + :pr:`15888` by `Hanmin Qin`_. + +- |Fix| :func:`metrics.plot_confusion_matrix` now colors the label color + correctly to maximize contrast with its background. :pr:`15936` by + `Thomas Fan`_ and :user:`DizietAsahi`. + +- |Fix| :func:`metrics.classification_report` does no longer ignore the + value of the ``zero_division`` keyword argument. :pr:`15879` + by :user:`Bibhash Chandra Mitra `. + +- |Fix| Fixed a bug in :func:`metrics.plot_confusion_matrix` to correctly + pass the `values_format` parameter to the :class:`ConfusionMatrixDisplay` + plot() call. :pr:`15937` by :user:`Stephen Blystone `. + +:mod:`sklearn.model_selection` +.............................. + +- |Fix| :class:`model_selection.GridSearchCV` and + :class:`model_selection.RandomizedSearchCV` accept scalar values provided in + `fit_params`. Change in 0.22 was breaking backward compatibility. + :pr:`15863` by :user:`Adrin Jalali ` and + :user:`Guillaume Lemaitre `. + +:mod:`sklearn.naive_bayes` +.......................... + +- |Fix| Removed `abstractmethod` decorator for the method `_check_X` in + :class:`naive_bayes.BaseNB` that could break downstream projects inheriting + from this deprecated public base class. :pr:`15996` by + :user:`Brigitta Sipőcz `. + +:mod:`sklearn.preprocessing` +............................ + +- |Fix| :class:`preprocessing.QuantileTransformer` now guarantees the + `quantiles_` attribute to be completely sorted in non-decreasing manner. + :pr:`15751` by :user:`Tirth Patel `. + +:mod:`sklearn.semi_supervised` +.............................. + +- |Fix| :class:`semi_supervised.LabelPropagation` and + :class:`semi_supervised.LabelSpreading` now allow callable kernel function to + return sparse weight matrix. + :pr:`15868` by :user:`Niklas Smedemark-Margulies `. + +:mod:`sklearn.utils` +.................... + +- |Fix| :func:`utils.check_array` now correctly converts pandas DataFrame with + boolean columns to floats. :pr:`15797` by `Thomas Fan`_. + +- |Fix| :func:`utils.check_is_fitted` accepts back an explicit ``attributes`` + argument to check for specific attributes as explicit markers of a fitted + estimator. When no explicit ``attributes`` are provided, only the attributes + that end with a underscore and do not start with double underscore are used + as "fitted" markers. The ``all_or_any`` argument is also no longer + deprecated. This change is made to restore some backward compatibility with + the behavior of this utility in version 0.21. :pr:`15947` by `Thomas Fan`_. + +:mod:`sklearn.multioutput` +.......................... + +- |Feature| :func:`multioutput.MultiOutputRegressor.fit` and + :func:`multioutput.MultiOutputClassifier.fit` now can accept `fit_params` + to pass to the `estimator.fit` method of each step. :issue:`15953` + :pr:`15959` by :user:`Ke Huang `. + .. _changes_0_22: Version 0.22.0 ============== -**In Development** +**December 3 2019** For a short description of the main highlights of the release, please refer to @@ -101,6 +258,7 @@ random sampling procedures. - :class:`linear_model.Ridge` when `X` is sparse. |Fix| - :class:`model_selection.StratifiedKFold` and any use of `cv=int` with a classifier. |Fix| +- :class:`cross_decomposition.CCA` when using scipy >= 1.3 |Fix| Details are listed in the changelog below. @@ -177,7 +335,7 @@ Changelog - |Fix| Fixed a bug in :class:`compose.ColumnTransformer` which failed to select the proper columns when using a boolean list, with NumPy older than 1.12. - :pr:`14510` by :user:`Guillaume Lemaitre `. + :pr:`14510` by `Guillaume Lemaitre`_. - |Fix| Fixed a bug in :class:`compose.TransformedTargetRegressor` which did not pass `**fit_params` to the underlying regressor. @@ -192,15 +350,25 @@ Changelog :mod:`sklearn.cross_decomposition` .................................. +- |Feature| :class:`cross_decomposition.PLSCanonical` and + :class:`cross_decomposition.PLSRegression` have a new function + ``inverse_transform`` to transform data to the original space. + :pr:`15304` by :user:`Jaime Ferrando Huertas `. + +- |Enhancement| :class:`decomposition.KernelPCA` now properly checks the + eigenvalues found by the solver for numerical or conditioning issues. This + ensures consistency of results across solvers (different choices for + ``eigen_solver``), including approximate solvers such as ``'randomized'`` and + ``'lobpcg'`` (see :issue:`12068`). + :pr:`12145` by :user:`Sylvain Marié ` + - |Fix| Fixed a bug where :class:`cross_decomposition.PLSCanonical` and :class:`cross_decomposition.PLSRegression` were raising an error when fitted with a target matrix `Y` in which the first column was constant. :issue:`13609` by :user:`Camila Williamson `. -- |Feature| :class:`cross_decomposition.PLSCanonical` and - :class:`cross_decomposition.PLSRegression` have a new function - ``inverse_transform`` to transform data to the original space`. - :pr:`15304` by :user:`Jaime Ferrando Huertas `. +- |Fix| :class:`cross_decomposition.CCA` now produces the same results with + scipy 1.3 and previous scipy versions. :pr:`15661` by `Thomas Fan`_. :mod:`sklearn.datasets` ....................... @@ -219,17 +387,21 @@ Changelog `weights` parameter, i.e. list or numpy.array, instead of list only. :pr:`14764` by :user:`Cat Chenal `. +- |Enhancement| The parameter `normalize` was added to + :func:`datasets.fetch_20newsgroups_vectorized`. + :pr:`14740` by :user:`Stéphan Tulkens ` + - |Fix| Fixed a bug in :func:`datasets.fetch_openml`, which failed to load an OpenML dataset that contains an ignored feature. :pr:`14623` by :user:`Sarra Habchi `. - - |Enhancement| The parameter `normalize` was added to - :func:`datasets.fetch_20newsgroups_vectorized`. - :pr:`14740` by :user:`Stéphan Tulkens ` - :mod:`sklearn.decomposition` ............................ +- |Efficiency| :class:`decomposition.NMF(solver='mu')` fitted on sparse input + matrices now uses batching to avoid briefly allocating an array with size + (#non-zero elements, n_components). :pr:`15257` by `Mart Willocx `_. + - |Enhancement| :func:`decomposition.dict_learning()` and :func:`decomposition.dict_learning_online()` now accept `method_max_iter` and pass it to :meth:`decomposition.sparse_encode`. @@ -251,22 +423,18 @@ Changelog underlying :class:`linear_model.LassoLars` when `algorithm='lasso_lars'`. :issue:`12650` by `Adrin Jalali`_. -- |Efficiency| :class:`decomposition.NMF(solver='mu')` fitted on sparse input - matrices now uses batching to avoid briefly allocating an array with size - (#non-zero elements, n_components). :pr:`15257` by `Mart Willocx `_. - :mod:`sklearn.dummy` .................... +- |Fix| :class:`dummy.DummyClassifier` now handles checking the existence + of the provided constant in multiouput cases. + :pr:`14908` by :user:`Martina G. Vilas `. + - |API| The default value of the `strategy` parameter in :class:`dummy.DummyClassifier` will change from `'stratified'` in version 0.22 to `'prior'` in 0.24. A FutureWarning is raised when the default value is used. :pr:`15382` by `Thomas Fan`_. -- |Fix| :class:`dummy.DummyClassifier` now handles checking the existence - of the provided constant in multiouput cases. - :pr:`14908` by :user:`Martina G. Vilas `. - - |API| The ``outputs_2d_`` attribute is deprecated in :class:`dummy.DummyClassifier` and :class:`dummy.DummyRegressor`. It is equivalent to ``n_outputs > 1``. :pr:`14933` by `Nicolas Hug`_ @@ -280,35 +448,44 @@ Changelog ` and :user:`Caio Oliveira ` and :pr:`15138` by :user:`Jon Cusick `.. -- Many improvements were made to +- |MajorFeature| Many improvements were made to :class:`ensemble.HistGradientBoostingClassifier` and :class:`ensemble.HistGradientBoostingRegressor`: - - |MajorFeature| Estimators now natively support dense data with missing + - |Feature| Estimators now natively support dense data with missing values both for training and predicting. They also support infinite values. :pr:`13911` and :pr:`14406` by `Nicolas Hug`_, `Adrin Jalali`_ and `Olivier Grisel`_. - |Feature| Estimators now have an additional `warm_start` parameter that enables warm starting. :pr:`14012` by :user:`Johann Faouzi `. + - |Feature| :func:`inspection.partial_dependence` and + :func:`inspection.plot_partial_dependence` now support the fast 'recursion' + method for both estimators. :pr:`13769` by `Nicolas Hug`_. - |Enhancement| for :class:`ensemble.HistGradientBoostingClassifier` the training loss or score is now monitored on a class-wise stratified subsample to preserve the class balance of the original training set. :pr:`14194` by :user:`Johann Faouzi `. - - |Feature| :func:`inspection.partial_dependence` and - :func:`inspection.plot_partial_dependence` now support the fast 'recursion' - method for both estimators. :pr:`13769` by `Nicolas Hug`_. - |Enhancement| :class:`ensemble.HistGradientBoostingRegressor` now supports the 'least_absolute_deviation' loss. :pr:`13896` by `Nicolas Hug`_. - |Fix| Estimators now bin the training and validation data separately to avoid any data leak. :pr:`13933` by `Nicolas Hug`_. - |Fix| Fixed a bug where early stopping would break with string targets. - :pr:`14710` by :user:`Guillaume Lemaitre `. + :pr:`14710` by `Guillaume Lemaitre`_. - |Fix| :class:`ensemble.HistGradientBoostingClassifier` now raises an error if ``categorical_crossentropy`` loss is given for a binary classification problem. :pr:`14869` by `Adrin Jalali`_. Note that pickles from 0.21 will not work in 0.22. +- |Enhancement| Addition of ``max_samples`` argument allows limiting + size of bootstrap samples to be less than size of dataset. Added to + :class:`ensemble.RandomForestClassifier`, + :class:`ensemble.RandomForestRegressor`, + :class:`ensemble.ExtraTreesClassifier`, + :class:`ensemble.ExtraTreesRegressor`. :pr:`14682` by + :user:`Matt Hancock ` and + :pr:`5963` by :user:`Pablo Duboue `. + - |Fix| :func:`ensemble.VotingClassifier.predict_proba` will no longer be present when `voting='hard'`. :pr:`14287` by `Thomas Fan`_. @@ -324,42 +501,30 @@ Changelog failing when the underlying estimators were not outputting consistent array dimensions. Note that it should be replaced by refactoring the common tests in the future. - :pr:`14305` by :user:`Guillaume Lemaitre `. + :pr:`14305` by `Guillaume Lemaitre`_. - |Fix| :class:`ensemble.AdaBoostClassifier` computes probabilities based on the decision function as in the literature. Thus, `predict` and `predict_proba` give consistent results. - :pr:`14114` by :user:`Guillaume Lemaitre `. - -- |API| ``presort`` is now deprecated in - :class:`ensemble.GradientBoostingClassifier` and - :class:`ensemble.GradientBoostingRegressor`, and the parameter has no effect. - Users are recommended to use :class:`ensemble.HistGradientBoostingClassifier` - and :class:`ensemble.HistGradientBoostingRegressor` instead. - :pr:`14907` by `Adrin Jalali`_. - -- |Enhancement| Addition of ``max_samples`` argument allows limiting - size of bootstrap samples to be less than size of dataset. Added to - :class:`ensemble.ForestClassifier`, - :class:`ensemble.ForestRegressor`, - :class:`ensemble.RandomForestClassifier`, - :class:`ensemble.RandomForestRegressor`, - :class:`ensemble.ExtraTreesClassifier`, - :class:`ensemble.ExtraTreesRegressor`, - :class:`ensemble.RandomTreesEmbedding`. :pr:`14682` by - :user:`Matt Hancock ` and - :pr:`5963` by :user:`Pablo Duboue `. + :pr:`14114` by `Guillaume Lemaitre`_. - |Fix| Stacking and Voting estimators now ensure that their underlying estimators are either all classifiers or all regressors. :class:`ensemble.StackingClassifier`, :class:`ensemble.StackingRegressor`, and :class:`ensemble.VotingClassifier` and :class:`VotingRegressor` now raise consistent error messages. - :pr:`15084` by :user:`Guillaume Lemaitre `. + :pr:`15084` by `Guillaume Lemaitre`_. - |Fix| :class:`ensemble.AdaBoostRegressor` where the loss should be normalized by the max of the samples with non-null weights only. - :pr:`14294` by :user:`Guillaume Lemaitre `. + :pr:`14294` by `Guillaume Lemaitre`_. + +- |API| ``presort`` is now deprecated in + :class:`ensemble.GradientBoostingClassifier` and + :class:`ensemble.GradientBoostingRegressor`, and the parameter has no effect. + Users are recommended to use :class:`ensemble.HistGradientBoostingClassifier` + and :class:`ensemble.HistGradientBoostingRegressor` instead. + :pr:`14907` by `Adrin Jalali`_. :mod:`sklearn.feature_extraction` ................................. @@ -375,11 +540,6 @@ Changelog :class:`feature_extraction.text.VectorizerMixin` can now be pickled. :pr:`14430` by :user:`Dillon Niederhut `. -- |API| Deprecated unused `copy` param for - :meth:`feature_extraction.text.TfidfVectorizer.transform` it will be - removed in v0.24. :pr:`14520` by - :user:`Guillem G. Subies `. - - |Fix| :func:`feature_extraction.text.strip_accents_unicode` now correctly removes accents from strings that are in NFKD normalized form. :pr:`15100` by :user:`Daniel Grady `. @@ -388,6 +548,11 @@ Changelog an `OverflowError` during the `transform` operation when producing a `scipy.sparse` matrix on large input data. :pr:`15463` by :user:`Norvan Sahiner `. +- |API| Deprecated unused `copy` param for + :meth:`feature_extraction.text.TfidfVectorizer.transform` it will be + removed in v0.24. :pr:`14520` by + :user:`Guillem G. Subies `. + :mod:`sklearn.feature_selection` ................................ @@ -408,7 +573,16 @@ Changelog :mod:`sklearn.gaussian_process` ............................... -- |Feature| :func:`gaussian_process.GaussianProcessClassifier.log_marginal_likelihood` +- |Feature| Gaussian process models on structured data: :class:`gaussian_process.GaussianProcessRegressor` + and :class:`gaussian_process.GaussianProcessClassifier` can now accept a list + of generic objects (e.g. strings, trees, graphs, etc.) as the ``X`` argument + to their training/prediction methods. + A user-defined kernel should be provided for computing the kernel matrix among + the generic objects, and should inherit from :class:`gaussian_process.kernels.GenericKernelMixin` + to notify the GPR/GPC model that it handles non-vectorial samples. + :pr:`15557` by :user:`Yu-Hang Tang `. + +- |Efficiency| :func:`gaussian_process.GaussianProcessClassifier.log_marginal_likelihood` and :func:`gaussian_process.GaussianProcessRegressor.log_marginal_likelihood` now accept a ``clone_kernel=True`` keyword argument. When set to ``False``, the kernel attribute is modified, but may result in a performance improvement. @@ -424,23 +598,19 @@ Changelog - |MajorFeature| Added :class:`impute.KNNImputer`, to impute missing values using k-Nearest Neighbors. :issue:`12852` by :user:`Ashim Bhattarai ` and - `Thomas Fan`_. - -- |Enhancement| Adds parameter `add_indicator` to :class:`impute.KNNImputer` - to get indicator of missing data. - :pr:`15010` by :user:`Guillaume Lemaitre `. + `Thomas Fan`_ and :pr:`15010` by `Guillaume Lemaitre`_. - |Feature| :class:`impute.IterativeImputer` has new `skip_compute` flag that is False by default, which, when True, will skip computation on features that have no missing values during the fit phase. :issue:`13773` by :user:`Sergey Feldman `. -- |Fix| :class:`impute.IterativeImputer` now works when there is only one feature. - By :user:`Sergey Feldman `. - - |Efficiency| :meth:`impute.MissingIndicator.fit_transform` avoid repeated computation of the masked matrix. :pr:`14356` by :user:`Harsh Soni `. +- |Fix| :class:`impute.IterativeImputer` now works when there is only one feature. + By :user:`Sergey Feldman `. + - |Fix| Fixed a bug in :class:`impute.IterativeImputer` where features where imputed in the reverse desired order with ``imputation_order`` either ``"ascending"`` or ``"descending"``. :pr:`15393` by @@ -467,7 +637,7 @@ Changelog and :class:`pipeline.Pipeline` containing :class:`compose.ColumnTransformer`. In addition :func:`inspection.plot_partial_dependence` will use the column names by default when a dataframe is passed. - :pr:`14028` and :pr:`15429` by :user:`Guillaume Lemaitre `. + :pr:`14028` and :pr:`15429` by `Guillaume Lemaitre`_. :mod:`sklearn.kernel_approximation` ................................... @@ -479,15 +649,15 @@ Changelog :mod:`sklearn.linear_model` ........................... +- |Efficiency| The 'liblinear' logistic regression solver is now faster and + requires less memory. + :pr:`14108`, :pr:`14170`, :pr:`14296` by :user:`Alex Henrie `. + - |Enhancement| :class:`linear_model.BayesianRidge` now accepts hyperparameters ``alpha_init`` and ``lambda_init`` which can be used to set the initial value of the maximization procedure in :term:`fit`. :pr:`13618` by :user:`Yoshihiro Uchida `. -- |Efficiency| The 'liblinear' logistic regression solver is now faster and - requires less memory. - :pr:`14108`, :pr:`14170`, :pr:`14296` by :user:`Alex Henrie `. - - |Fix| :class:`linear_model.Ridge` now correctly fits an intercept when `X` is sparse, `solver="auto"` and `fit_intercept=True`, because the default solver in this configuration has changed to `sparse_cg`, which can fit an intercept @@ -495,26 +665,26 @@ Changelog - |Fix| :class:`linear_model.Ridge` with `solver='sag'` now accepts F-ordered and non-contiguous arrays and makes a conversion instead of failing. - :pr:`14458` by :user:`Guillaume Lemaitre `. + :pr:`14458` by `Guillaume Lemaitre`_. - |Fix| :class:`linear_model.LassoCV` no longer forces ``precompute=False`` when fitting the final model. :pr:`14591` by `Andreas Müller`_. -- |FIX| :class:`linear_model.RidgeCV` and :class:`linear_model.RidgeClassifierCV` +- |Fix| :class:`linear_model.RidgeCV` and :class:`linear_model.RidgeClassifierCV` now correctly scores when `cv=None`. :pr:`14864` by :user:`Venkatachalam N `. -- |FIX| Fixed a bug in :class:`linear_model.LogisticRegressionCV` where the +- |Fix| Fixed a bug in :class:`linear_model.LogisticRegressionCV` where the ``scores_``, ``n_iter_`` and ``coefs_paths_`` attribute would have a wrong ordering with ``penalty='elastic-net'``. :pr:`15044` by `Nicolas Hug`_ -- |FIX| :class:`linear_model.MultiTaskLassoCV` and +- |Fix| :class:`linear_model.MultiTaskLassoCV` and :class:`linear_model.MultiTaskElasticNetCV` with X of dtype int and `fit_intercept=True`. :pr:`15086` by :user:`Alex Gramfort `. -- |FIX| The liblinear solver now supports ``sample_weight``. - :pr:`15038` by :user:`Guillaume Lemaitre `. +- |Fix| The liblinear solver now supports ``sample_weight``. + :pr:`15038` by `Guillaume Lemaitre`_. :mod:`sklearn.manifold` ....................... @@ -533,9 +703,6 @@ Changelog ``method="barnes-hut"`` by computing the gradient in parallel. :pr:`13213` by :user:`Thomas Moreau ` -- |API| Deprecate ``training_data_`` unused attribute in - :class:`manifold.Isomap`. :issue:`10482` by `Tom Dupre la Tour`_. - - |Fix| Fixed a bug where :func:`manifold.spectral_embedding` (and therefore :class:`manifold.SpectralEmbedding` and :class:`cluster.SpectralClustering`) computed wrong eigenvalues with ``eigen_solver='amg'`` when @@ -547,9 +714,16 @@ Changelog :issue:`13393` by :user:`Andrew Knyazev ` :pr:`13707` by :user:`Scott White ` +- |API| Deprecate ``training_data_`` unused attribute in + :class:`manifold.Isomap`. :issue:`10482` by `Tom Dupre la Tour`_. + :mod:`sklearn.metrics` ...................... +- |MajorFeature| :func:`metrics.plot_roc_curve` has been added to plot roc + curves. This function introduces the visualization API described in + the :ref:`User Guide `. :pr:`14357` by `Thomas Fan`_. + - |Feature| Added a new parameter ``zero_division`` to multiple classification metrics: :func:`precision_score`, :func:`recall_score`, :func:`f1_score`, :func:`fbeta_score`, :func:`precision_recall_fscore_support`, @@ -557,10 +731,6 @@ Changelog ill-defined metrics. :pr:`14900` by :user:`Marc Torrellas Socastro `. -- |MajorFeature| :func:`metrics.plot_roc_curve` has been added to plot roc - curves. This function introduces the visualization API described in - the :ref:`User Guide `. :pr:`14357` by `Thomas Fan`_. - - |Feature| Added the :func:`metrics.pairwise.nan_euclidean_distances` metric, which calculates euclidean distances in the presence of missing values. :issue:`12852` by :user:`Ashim Bhattarai ` and `Thomas Fan`_. @@ -573,6 +743,9 @@ Changelog - |Feature| :func:`metrics.plot_precision_recall_curve` has been added to plot precision recall curves. :pr:`14936` by `Thomas Fan`_. +- |Feature| :func:`metrics.plot_confusion_matrix` has been added to plot + confusion matrices. :pr:`15083` by `Thomas Fan`_. + - |Feature| Added multiclass support to :func:`metrics.roc_auc_score` with corresponding scorers `'roc_auc_ovr'`, `'roc_auc_ovo'`, `'roc_auc_ovr_weighted'`, and `'roc_auc_ovo_weighted'`. @@ -588,6 +761,10 @@ Changelog :pr:`13938` by :user:`Christian Lorentzen ` and `Roman Yurchak`_. +- |Efficiency| Improved performance of + :func:`metrics.pairwise.manhattan_distances` in the case of sparse matrices. + :pr:`15049` by `Paolo Toccaceli `. + - |Enhancement| The parameter ``beta`` in :func:`metrics.fbeta_score` is updated to accept the zero and `float('+inf')` value. :pr:`13231` by :user:`Dong-hee Na `. @@ -599,23 +776,11 @@ Changelog - |Enhancement| Allow computing averaged metrics in the case of no true positives. :pr:`14595` by `Andreas Müller`_. -- |Fix| Raise a ValueError in :func:`metrics.silhouette_score` when a - precomputed distance matrix contains non-zero diagonal entries. - :pr:`12258` by :user:`Stephen Tierney `. - - |Enhancement| Multilabel metrics now supports list of lists as input. :pr:`14865` :user:`Srivatsan Ramesh `, :user:`Herilalaina Rakotoarison `, :user:`Léonard Binet `. -- |API| ``scoring="neg_brier_score"`` should be used instead of - ``scoring="brier_score_loss"`` which is now deprecated. - :pr:`14898` by :user:`Stefan Matcovici `. - -- |Efficiency| Improved performance of - :func:`metrics.pairwise.manhattan_distances` in the case of sparse matrices. - :pr:`15049` by `Paolo Toccaceli `. - - |Enhancement| :func:`metrics.median_absolute_error` now supports ``multioutput`` parameter. :pr:`14732` by :user:`Agamemnon Krasoulis `. @@ -624,6 +789,19 @@ Changelog used as the :term:`scoring` parameter of model-selection tools. :pr:`14417` by `Thomas Fan`_. +- |Enhancement| :func:`metrics.confusion_matrix` accepts a parameters + `normalize` allowing to normalize the confusion matrix by column, rows, or + overall. + :pr:`15625` by `Guillaume Lemaitre `. + +- |Fix| Raise a ValueError in :func:`metrics.silhouette_score` when a + precomputed distance matrix contains non-zero diagonal entries. + :pr:`12258` by :user:`Stephen Tierney `. + +- |API| ``scoring="neg_brier_score"`` should be used instead of + ``scoring="brier_score_loss"`` which is now deprecated. + :pr:`14898` by :user:`Stefan Matcovici `. + :mod:`sklearn.model_selection` .............................. @@ -644,16 +822,16 @@ Changelog where one test set could be `n_classes` larger than another. Test sets should now be near-equally sized. :pr:`14704` by `Joel Nothman`_. +- |Fix| The `cv_results_` attribute of :class:`model_selection.GridSearchCV` + and :class:`model_selection.RandomizedSearchCV` now only contains unfitted + estimators. This potentially saves a lot of memory since the state of the + estimators isn't stored. :pr:`#15096` by `Andreas Müller`_. + - |API| :class:`model_selection.KFold` and :class:`model_selection.StratifiedKFold` now raise a warning if `random_state` is set but `shuffle` is False. This will raise an error in 0.24. -- |Fix| The `cv_results_` attribute of :class:`model_selection.GridSearchCV` - and :class:`model_selection.RandomizedSearchCV` now only contains unfitted - estimators. This potentially saves a lot of memory since the state of the - estimators isn't stored. :pr:`#15096` by :user:`Andreas Müller `. - :mod:`sklearn.multioutput` .......................... @@ -720,13 +898,13 @@ Changelog the final estimator does. :pr:`13806` by :user:`Anaël Beaugnon `. +- |Fix| The `fit` in :class:`~pipeline.FeatureUnion` now accepts `fit_params` + to pass to the underlying transformers. :pr:`15119` by `Adrin Jalali`_. + - |API| `None` as a transformer is now deprecated in :class:`pipeline.FeatureUnion`. Please use `'drop'` instead. :pr:`15053` by `Thomas Fan`_. -- |Fix| The `fit` in :class:`~pipeline.FeatureUnion` now accepts `fit_params` - to pass to the underlying transformers. :pr:`15119` by `Adrin Jalali`_. - :mod:`sklearn.preprocessing` ............................ @@ -744,7 +922,7 @@ Changelog :pr:`14336` by :user:`Gregory Dexter `. :mod:`sklearn.model_selection` -.................. +.............................. - |Fix| :class:`model_selection.GridSearchCV` and `model_selection.RandomizedSearchCV` now supports the @@ -755,7 +933,6 @@ Changelog :pr:`13925` by :user:`Isaac S. Robson ` and :pr:`15524` by :user:`Xun Tang `. - :mod:`sklearn.svm` .................. @@ -772,7 +949,7 @@ Changelog - |Fix| :class:`svm.SVC`, :class:`svm.SVR`, :class:`svm.NuSVR` and :class:`svm.OneClassSVM` when received values negative or zero for parameter ``sample_weight`` in method fit(), generated an - invalid model. This behavior occured only in some border scenarios. + invalid model. This behavior occurred only in some border scenarios. Now in these cases, fit() will fail with an Exception. :pr:`14286` by :user:`Alex Shacked `. @@ -783,8 +960,8 @@ Changelog - |Fix| fixed a bug in :class:`BaseLibSVM._sparse_fit` where n_SV=0 raised a ZeroDivisionError. :pr:`14894` by :user:`Danna Naser `. -- |FIX| The liblinear solver now supports ``sample_weight``. - :pr:`15038` by :user:`Guillaume Lemaitre `. +- |Fix| The liblinear solver now supports ``sample_weight``. + :pr:`15038` by `Guillaume Lemaitre`_. :mod:`sklearn.tree` @@ -797,7 +974,6 @@ Changelog :class:`ensemble.RandomForestRegressor`, :class:`ensemble.ExtraTreesClassifier`, :class:`ensemble.ExtraTreesRegressor`, - :class:`ensemble.RandomTreesEmbedding`, :class:`ensemble.GradientBoostingClassifier`, and :class:`ensemble.GradientBoostingRegressor`. :pr:`12887` by `Thomas Fan`_. @@ -825,22 +1001,7 @@ Changelog :func:`~utils.estimator_checks.parametrize_with_checks`, to parametrize estimator checks for a list of estimators. :pr:`14381` by `Thomas Fan`_. -- |API| The following utils have been deprecated and are now private: - - - ``utils.choose_check_classifiers_labels`` - - ``utils.enforce_estimator_tags_y`` - - ``utils.optimize.newton_cg`` - - ``utils.random.random_choice_csc`` - - ``utils.safe_indexing`` - - ``utils.mocking`` - - ``utils.fast_dict`` - - ``utils.seq_dataset`` - - ``utils.weight_vector`` - - ``utils.fixes.parallel_helper`` (removed) - - All of ``utils.testing`` except for ``all_estimators`` which is now in - ``utils``. - -- A new random variable, :class:`utils.fixes.loguniform` implements a +- |Feature| A new random variable, :class:`utils.fixes.loguniform` implements a log-uniform random variable (e.g., for use in RandomizedSearchCV). For example, the outcomes ``1``, ``10`` and ``100`` are all equally likely for ``loguniform(1, 100)``. See :issue:`11232` by @@ -851,7 +1012,7 @@ Changelog ``axis`` parameter to index array-like across rows and columns. The column indexing can be done on NumPy array, SciPy sparse matrix, and Pandas DataFrame. An additional refactoring was done. :pr:`14035` and :pr:`14475` - by :user:`Guillaume Lemaitre `. + by `Guillaume Lemaitre`_. - |Enhancement| :func:`utils.extmath.safe_sparse_dot` works between 3D+ ndarray and sparse matrix. @@ -873,6 +1034,18 @@ Changelog - ``mocking.CheckingClassifier`` - ``optimize.newton_cg`` - ``random.random_choice_csc`` + - ``utils.choose_check_classifiers_labels`` + - ``utils.enforce_estimator_tags_y`` + - ``utils.optimize.newton_cg`` + - ``utils.random.random_choice_csc`` + - ``utils.safe_indexing`` + - ``utils.mocking`` + - ``utils.fast_dict`` + - ``utils.seq_dataset`` + - ``utils.weight_vector`` + - ``utils.fixes.parallel_helper`` (removed) + - All of ``utils.testing`` except for ``all_estimators`` which is now in + ``utils``. :mod:`sklearn.isotonic` .................................. @@ -881,10 +1054,13 @@ Changelog when `X.dtype == 'float32'` and `X.dtype != y.dtype`. :pr:`14902` by :user:`Lucas `. - Miscellaneous ............. +- |Fix| Port `lobpcg` from SciPy which implement some bug fixes but only + available in 1.3+. + :pr:`13609` and :pr:`14971` by `Guillaume Lemaitre`_. + - |API| Scikit-learn now converts any input data structure implementing a duck array to a numpy array (using ``__array__``) to ensure consistent behavior instead of relying on ``__array_function__`` (see `NEP 18 @@ -895,10 +1071,6 @@ Miscellaneous using a non-fitted estimators are now more uniform. :pr:`13013` by :user:`Agamemnon Krasoulis `. -- |Fix| Port `lobpcg` from SciPy which implement some bug fixes but only - available in 1.3+. - :pr:`13609` and :pr:`14971` by :user:`Guillaume Lemaitre `. - Changes to estimator checks --------------------------- @@ -935,5 +1107,66 @@ These changes mostly affect library developers. - |Fix| Added ``check_transformer_data_not_an_array`` to checks where missing - |Fix| The estimators tags resolution now follows the regular MRO. They used - to be overridable only once. :pr:`14884` by :user:`Andreas Müller - `. + to be overridable only once. :pr:`14884` by `Andreas Müller`_. + + +Code and Documentation Contributors +----------------------------------- + +Thanks to everyone who has contributed to the maintenance and improvement of the +project since version 0.20, including: + +Aaron Alphonsus, Abbie Popa, Abdur-Rahmaan Janhangeer, abenbihi, Abhinav Sagar, +Abhishek Jana, Abraham K. Lagat, Adam J. Stewart, Aditya Vyas, Adrin Jalali, +Agamemnon Krasoulis, Alec Peters, Alessandro Surace, Alexandre de Siqueira, +Alexandre Gramfort, alexgoryainov, Alex Henrie, Alex Itkes, alexshacked, Allen +Akinkunle, Anaël Beaugnon, Anders Kaseorg, Andrea Maldonado, Andrea Navarrete, +Andreas Mueller, Andreas Schuderer, Andrew Nystrom, Angela Ambroz, Anisha +Keshavan, Ankit Jha, Antonio Gutierrez, Anuja Kelkar, Archana Alva, +arnaudstiegler, arpanchowdhry, ashimb9, Ayomide Bamidele, Baran Buluttekin, +barrycg, Bharat Raghunathan, Bill Mill, Biswadip Mandal, blackd0t, Brian G. +Barkley, Brian Wignall, Bryan Yang, c56pony, camilaagw, cartman_nabana, +catajara, Cat Chenal, Cathy, cgsavard, Charles Vesteghem, Chiara Marmo, Chris +Gregory, Christian Lorentzen, Christos Aridas, Dakota Grusak, Daniel Grady, +Daniel Perry, Danna Naser, DatenBergwerk, David Dormagen, deeplook, Dillon +Niederhut, Dong-hee Na, Dougal J. Sutherland, DrGFreeman, Dylan Cashman, +edvardlindelof, Eric Larson, Eric Ndirangu, Eunseop Jeong, Fanny, +federicopisanu, Felix Divo, flaviomorelli, FranciDona, Franco M. Luque, Frank +Hoang, Frederic Haase, g0g0gadget, Gabriel Altay, Gabriel do Vale Rios, Gael +Varoquaux, ganevgv, gdex1, getgaurav2, Gideon Sonoiya, Gordon Chen, gpapadok, +Greg Mogavero, Grzegorz Szpak, Guillaume Lemaitre, Guillem García Subies, +H4dr1en, hadshirt, Hailey Nguyen, Hanmin Qin, Hannah Bruce Macdonald, Harsh +Mahajan, Harsh Soni, Honglu Zhang, Hossein Pourbozorg, Ian Sanders, Ingrid +Spielman, J-A16, jaehong park, Jaime Ferrando Huertas, James Hill, James Myatt, +Jay, jeremiedbb, Jérémie du Boisberranger, jeromedockes, Jesper Dramsch, Joan +Massich, Joanna Zhang, Joel Nothman, Johann Faouzi, Jonathan Rahn, Jon Cusick, +Jose Ortiz, Kanika Sabharwal, Katarina Slama, kellycarmody, Kennedy Kang'ethe, +Kensuke Arai, Kesshi Jordan, Kevad, Kevin Loftis, Kevin Winata, Kevin Yu-Sheng +Li, Kirill Dolmatov, Kirthi Shankar Sivamani, krishna katyal, Lakshmi Krishnan, +Lakshya KD, LalliAcqua, lbfin, Leland McInnes, Léonard Binet, Loic Esteve, +loopyme, lostcoaster, Louis Huynh, lrjball, Luca Ionescu, Lutz Roeder, +MaggieChege, Maithreyi Venkatesh, Maltimore, Maocx, Marc Torrellas, Marie +Douriez, Markus, Markus Frey, Martina G. Vilas, Martin Oywa, Martin Thoma, +Masashi SHIBATA, Maxwell Aladago, mbillingr, m-clare, Meghann Agarwal, m.fab, +Micah Smith, miguelbarao, Miguel Cabrera, Mina Naghshhnejad, Ming Li, motmoti, +mschaffenroth, mthorrell, Natasha Borders, nezar-a, Nicolas Hug, Nidhin +Pattaniyil, Nikita Titov, Nishan Singh Mann, Nitya Mandyam, norvan, +notmatthancock, novaya, nxorable, Oleg Stikhin, Oleksandr Pavlyk, Olivier +Grisel, Omar Saleem, Owen Flanagan, panpiort8, Paolo, Paolo Toccaceli, Paresh +Mathur, Paula, Peng Yu, Peter Marko, pierretallotte, poorna-kumar, pspachtholz, +qdeffense, Rajat Garg, Raphaël Bournhonesque, Ray, Ray Bell, Rebekah Kim, Reza +Gharibi, Richard Payne, Richard W, rlms, Robert Juergens, Rok Mihevc, Roman +Feldbauer, Roman Yurchak, R Sanjabi, RuchitaGarde, Ruth Waithera, Sackey, Sam +Dixon, Samesh Lakhotia, Samuel Taylor, Sarra Habchi, Scott Gigante, Scott +Sievert, Scott White, Sebastian Pölsterl, Sergey Feldman, SeWook Oh, she-dares, +Shreya V, Shubham Mehta, Shuzhe Xiao, SimonCW, smarie, smujjiga, Sönke +Behrends, Soumirai, Sourav Singh, stefan-matcovici, steinfurt, Stéphane +Couvreur, Stephan Tulkens, Stephen Cowley, Stephen Tierney, SylvainLan, +th0rwas, theoptips, theotheo, Thierno Ibrahima DIOP, Thomas Edwards, Thomas J +Fan, Thomas Moreau, Thomas Schmitt, Tilen Kusterle, Tim Bicker, Timsaur, Tim +Staley, Tirth Patel, Tola A, Tom Augspurger, Tom Dupré la Tour, topisan, Trevor +Stephens, ttang131, Urvang Patel, Vathsala Achar, veerlosar, Venkatachalam N, +Victor Luzgin, Vincent Jeanselme, Vincent Lostanlen, Vladimir Korolev, +vnherdeiro, Wenbo Zhao, Wendy Hu, willdarnell, William de Vazelhes, +wolframalpha, xavier dupré, xcjason, x-martian, xsat, xun-tang, Yinglr, +yokasre, Yu-Hang "Maxin" Tang, Yulia Zamriy, Zhao Feng diff --git a/doc/whats_new/v0.23.rst b/doc/whats_new/v0.23.rst new file mode 100644 index 0000000000000..d0c568956a353 --- /dev/null +++ b/doc/whats_new/v0.23.rst @@ -0,0 +1,421 @@ +.. include:: _contributors.rst + +.. currentmodule:: sklearn + +.. _changes_0_23: + +Version 0.23.0 +============== + +**In Development** + + +.. include:: changelog_legend.inc + +Put the changes in their relevant module. + +Changed models +-------------- + +The following estimators and functions, when fit with the same data and +parameters, may produce different models from the previous version. This often +occurs due to changes in the modelling logic (bug fixes or enhancements), or in +random sampling procedures. + +- :class:`ensemble.BaggingClassifier`, :class:`ensemble.BaggingRegressor`, + and :class:`ensemble.IsolationForest`. |Fix| + +Details are listed in the changelog below. + +(While we are trying to better inform users by providing this information, we +cannot assure that this list is complete.) + +Changelog +--------- + +.. + Entries should be grouped by module (in alphabetic order) and prefixed with + one of the labels: |MajorFeature|, |Feature|, |Efficiency|, |Enhancement|, + |Fix| or |API| (see whats_new.rst for descriptions). + Entries should be ordered by those labels (e.g. |Fix| after |Efficiency|). + Changes not specific to a module should be listed under *Multiple Modules* + or *Miscellaneous*. + Entries should end with: + :pr:`123456` by :user:`Joe Bloggs `. + where 123456 is the *pull request* number, not the issue number. + +:mod:`sklearn.cluster` +...................... + +- |Enhancement| :class:`cluster.AgglomerativeClustering` has a faster and more + more memory efficient implementation of single linkage clustering. + :pr:`11514` by :user:`Leland McInnes `. +- |Fix| :class:`cluster.KMeans` with ``algorithm="elkan"`` now converges with + ``tol=0`` as with the default ``algorithm="full"``. :pr:`16075` by + :user:`Erich Schubert `. + +- |Efficiency| :class:`cluster.Birch` implementation of the predict method + avoids high memory footprint by calculating the distances matrix using + a chunked scheme. + :pr:`16149` by :user:`Jeremie du Boisberranger ` and + :user:`Alex Shacked `. + +- |Fix| Fixed a bug in :class:`cluster.Birch` where the `n_clusters` parameter + could not have a `np.int64` type. :pr:`16484` + by :user:`Jeremie du Boisberranger `. + +- |API| The ``n_jobs`` parameter of :class:`cluster.KMeans`, + :class:`cluster.SpectralCoclustering` and + :class:`cluster.SpectralBiclustering` is deprecated. They now use OpenMP + based parallelism. For more details on how to control the number of threads, + please refer to our :ref:`parallelism` notes. :pr:`11950` by + :user:`Jeremie du Boisberranger `. + +- |API| The ``precompute_distances`` parameter of :class:`cluster.KMeans` is + deprecated. It has no effect. :pr:`11950` by + :user:`Jeremie du Boisberranger `. + +- |Efficiency| The critical parts of :class:`cluster.KMeans` have a more + optimized implementation. Parallelism is now over the data instead of over + initializations allowing better scalability. :pr:`11950` by + :user:`Jeremie du Boisberranger `. + +- |Enhancement| :class:`cluster.KMeans` now supports sparse data when + `solver = "elkan"`. :pr:`11950` by + :user:`Jeremie du Boisberranger `. + +:mod:`sklearn.compose` +...................... + +- |Fix| :class:`compose.ColumnTransformer` method ``get_feature_names`` now + returns correct results when one of the transformer steps applies on an + empty list of columns :pr:`15963` by `Roman Yurchak`_. + +- |Efficiency| :class:`compose.ColumnTransformer` is now faster when working + with dataframes and strings are used to specific subsets of data for + transformers. :pr:`16431` by `Thomas Fan`_. + +- |Fix| :func:`compose.ColumnTransformer.fit` will error when selecting + a column name that is not unique in the dataframe. :pr:`16431` by + `Thomas Fan`_. + +:mod:`sklearn.datasets` +....................... + +- |Enhancement| Added ``return_centers`` parameter in + :func:`datasets.make_blobs`, which can be used to return + centers for each cluster. + :pr:`15709` by :user:`` and + :user:`Venkatachalam N `. + +- |Enhancement| Functions :func:`datasets.make_circles` and + :func:`datasets.make_moons` now accept two-element tuple. + :pr:`15707` by :user:`Maciej J Mikulski `. + +- |Feature| :func:`datasets.fetch_california_housing` now supports + heterogeneous data using pandas by setting `as_frame=True`. :pr:`15950` + by :user:`Stephanie Andrews ` and + :user:`Reshama Shaikh `. + +- |Feature| embedded dataset loaders :func:`load_breast_cancer`, + :func:`load_diabetes`, :func:`load_digits`, :func:`load_iris`, + :func:`load_linnerud` and :func:`load_wine` now support loading as a pandas + ``DataFrame`` by setting `as_frame=True`. :pr:`15980` by :user:`wconnell` and + :user:`Reshama Shaikh `. + +- |Fix| :func:`datasets.make_multilabel_classification` now generates + `ValueError` for arguments `n_classes < 1` OR `length < 1`. + :pr:`16006` by :user:`Rushabh Vasani `. + +:mod:`sklearn.decomposition` +............................ + +- |Fix| :class:`decomposition.PCA` with a float `n_components` parameter, will + exclusively choose the components that explain the variance greater than + `n_components`. :pr:`15669` by :user:`Krishna Chaitanya ` +- |Fix| :func:`decomposition._pca._assess_dimension` now correctly handles small + eigenvalues. :pr: `4441` by :user:`Lisa Schwetlick `, and + :user:`Gelavizh Ahmadi ` and + :user:`Marija Vlajic Wheeler `. + +- |Enhancement| :class:`decomposition.NMF` and + :func:`decomposition.non_negative_factorization` now preserves float32 dtype. + :pr:`16280` by :user:`Jeremie du Boisberranger `. + +- |Fix| :class:`decomposition.KernelPCA` method ``inverse_transform`` now + applies the correct inverse transform to the transformed data. :pr:`16655` + by :user:`Lewis Ball `. + +:mod:`sklearn.ensemble` +....................... + +- |MajorFeature| :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` now support + :term:`sample_weight`. :pr:`14696` by `Adrin Jalali`_ and `Nicolas Hug`_. + +- |API| Added boolean `verbose` flag to classes: + :class:`ensemble.VotingClassifier` and :class:`ensemble.VotingRegressor`. + :pr:`15991` by :user:`Sam Bail `, + :user:`Hanna Bruce MacDonald `, + :user:`Reshama Shaikh `, and + :user:`Chiara Marmo `. + +- |API| Fixed a bug in :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegrerssor` that would not respect the + `max_leaf_nodes` parameter if the criteria was reached at the same time as + the `max_depth` criteria. :pr:`16183` by `Nicolas Hug`_. + +- |Fix| Changed the convention for `max_depth` parameter of + :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor`. The depth now corresponds to + the number of edges to go from the root to the deepest leaf. + Stumps (trees with one split) are now allowed. + :pr: `16182` by :user:`Santhosh B ` + +- |Feature| Early stopping in + :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` is now determined with a + new `early_stopping` parameter instead of `n_iter_no_change`. Default value + is 'auto', which enables early stopping if there are at least 10,000 + samples in the training set. :pr:`14516` by :user:`Johann Faouzi + `. + +- |Fix| Fixed a bug in :class:`ensemble.BaggingClassifier`, + :class:`ensemble.BaggingRegressor` and :class:`ensemble.IsolationForest` + where the attribute `estimators_samples_` did not generate the proper indices + used during `fit`. + :pr:`16437` by :user:`Jin-Hwan CHO `. + +- |Fix| Fixed a bug in :class:`ensemble.StackingClassifier` and + :class:`ensemble.StackingRegressor` where the `sample_weight` + argument was not being passed to `cross_val_predict` when + evaluating the base estimators on cross-validation folds + to obtain the input to the meta estimator. + :pr:`16539` by :user:`Bill DeRose `. + +:mod:`sklearn.feature_extraction` +................................. + +- |Efficiency| :class:`feature_extraction.text.CountVectorizer` now sorts + features after pruning them by document frequency. This improves performances + for datasets with large vocabularies combined with ``min_df`` or ``max_df``. + :pr:`15834` by :user:`Santiago M. Mola `. + + +- |Enhancement| Added support for multioutput data in + :class:`feature_selection.RFE` and :class:`feature_selection.RFECV`. + :pr:`16103` by :user:`Divyaprabha M `. + +:mod:`sklearn.gaussian_process` +............................... + +- |Enhancement| :func:`gaussian_process.kernels.Matern` returns the RBF kernel when ``nu=np.inf``. + :pr:`15503` by :user:`Sam Dixon` . + +- |Fix| Fixed bug in :class:`gaussian_process.GaussianProcessRegressor` that + caused predicted standard deviations to only be between 0 and 1 when + WhiteKernel is not used. :pr:`15782` + by :user:`plgreenLIRU`. + +:mod:`sklearn.impute` +..................... + +- |Enhancement| :class:`impute.IterativeImputer` accepts both scalar and array-like inputs for + ``max_value`` and ``min_value``. Array-like inputs allow a different max and min to be specified + for each feature. :pr:`16403` by :user:`Narendra Mukherjee `. + +:mod:`sklearn.inspection` +......................... + +- |Feature| :func:`inspection.partial_dependence` and + :func:`inspection.plot_partial_dependence` now support the fast 'recursion' + method for :class:`ensemble.RandomForestRegressor` and + :class:`tree.DecisionTreeRegressor`. :pr:`15864` by + `Nicolas Hug`_. + +:mod:`sklearn.linear_model` +........................... + +- |MajorFeature| Added generalized linear models (GLM) with non normal error + distributions, including :class:`linear_model.PoissonRegressor`, + :class:`linear_model.GammaRegressor` and :class:`linear_model.TweedieRegressor` + which use Poisson, Gamma and Tweedie distributions respectively. + :pr:`14300` by :user:`Christian Lorentzen `, `Roman Yurchak`_, + and `Olivier Grisel`_. + +- |Feature| Support of `sample_weight` in :class:`linear_model.ElasticNet` and + :class:`linear_model:Lasso` for dense feature matrix `X`. + :pr:`15436` by :user:`Christian Lorentzen `. + +- |Fix| Fixed a bug where if a `sample_weight` parameter was passed to the fit + method of :class:`linear_model.RANSACRegressor`, it would not be passed to + the wrapped `base_estimator` during the fitting of the final model. + :pr:`15573` by :user:`Jeremy Alexandre `. + +- |Efficiency| :class:`linear_model.RidgeCV` and + :class:`linear_model.RidgeClassifierCV` now does not allocate a + potentially large array to store dual coefficients for all hyperparameters + during its `fit`, nor an array to store all error or LOO predictions unless + `store_cv_values` is `True`. + :pr:`15652` by :user:`Jérôme Dockès `. + +- |Fix| add `best_score_` attribute to :class:`linear_model.RidgeCV` and + :class:`linear_model.RidgeClassifierCV`. + :pr:`15653` by :user:`Jérôme Dockès `. + +- |Fix| Fixed a bug in :class:`linear_model.RidgeClassifierCV` to pass a + specific scoring strategy. Before the internal estimator outputs score + instead of predictions. + :pr:`14848` by :user:`Venkatachalam N `. + +- |API| Deprecated public attributes `standard_coef_`, `standard_intercept_`, + `average_coef_`, and `average_intercept_` in + :class:`linear_model.SGDClassifier`, + :class:`linear_model.SGDRegressor`, + :class:`linear_model.PassiveAggressiveClassifier`, + :class:`linear_model.PassiveAggressiveRegressor`. + :pr:`16261` by :user:`Carlos Brandt `. + +- |Fix| :class:`linear_model.LogisticRegression` will now avoid an unnecessary + iteration when `solver='newton-cg'` by checking for inferior or equal instead + of strictly inferior for maximum of `absgrad` and `tol` in `utils.optimize._newton_cg`. + :pr:`16266` by :user:`Rushabh Vasani `. + +:mod:`sklearn.metrics` +...................... + +- |API| Changed the formatting of values in + :meth:`metrics.ConfusionMatrixDisplay.plot` and + :func:`metrics.plot_confusion_matrix` to pick the shorter format (either '2g' + or 'd'). :pr:`16159` by :user:`Rick Mackenbach ` and + `Thomas Fan`_. + +- |Enhancement| :func:`metrics.pairwise.pairwise_distances_chunked` now allows + its ``reduce_func`` to not have a return value, enabling in-place operations. + :pr:`16397` by `Joel Nothman`_. + +- |Fix| Fixed a bug in :func:`metrics.mean_squared_error` to not ignore + argument `squared` when argument `multioutput='raw_values'`. + :pr:`16323` by :user:`Rushabh Vasani ` + +- |Fix| Fixed a bug in :func:`metrics.mutual_info_score` where negative + scores could be returned. :pr:`16362` by `Thomas Fan`_. + +- |Fix| Fixed a bug in :func:`metrics.confusion_matrix` that would raise + an error when `y_true` and `y_pred` were length zero and `labels` was + not `None`. In addition, we raise an error when an empty list is given to + the `labels` parameter. + :pr:`16442` by `Kyle Parsons `. + +:mod:`sklearn.model_selection` +.............................. + +- |Enhancement| :class:`model_selection.GridSearchCV` and + :class:`model_selection.RandomizedSearchCV` yields stack trace information + in fit failed warning messages in addition to previously emitted + type and details. + :pr:`15622` by :user:`Gregory Morse `. + +- |Fix| :func: `cross_val_predict` supports `method="predict_proba"` + when `y=None`. + :pr:`15918` by :user:`Luca Kubin `. + +- |Fix| :func:`model_selection.fit_grid_point` is deprecated in 0.23 and will + be removed in 0.25. :pr:`16401` by + :user:`Arie Pratama Sutiono ` + +:mod:`sklearn.multioutput` +.......................... + +- |Enhancement| :class:`multioutput.RegressorChain` now supports `fit_params` + for `base_estimator` during `fit`. + :pr:`16111` by :user:`Venkatachalam N `. + +:mod:`sklearn.naive_bayes` +............................. + +- |Fix| A correctly formatted error message is shown in + :class:`naive_bayes.CategoricalNB` when the number of features in the input + differs between `predict` and `fit`. + :pr:`16090` by :user:`Madhura Jayaratne `. + +:mod:`sklearn.neural_network` +............................. + +- |Fix| Increases the numerical stability of the logistic loss function in + :class:`neural_network.MLPClassifier` by clipping the probabilities. + :pr:`16117` by `Thomas Fan`_. + +:mod:`sklearn.preprocessing` +............................ + +- |Efficiency| :class:`preprocessing.OneHotEncoder` is now faster at + transforming. :pr:`15762` by `Thomas Fan`_. + +- |Feature| argument `drop` of :class:`preprocessing.OneHotEncoder` + will now accept value 'if_binary' and will drop the first category of + each feature with two categories. :pr:`#16245` + by :user:`Rushabh Vasani `. + +- |Fix| Fix a bug in :class:`preprocessing.StandardScaler` which was incorrectly + computing statistics when calling `partial_fit` on sparse inputs. + :pr:`16466` by :user:`Guillaume Lemaitre `. + +- |Fix| Fix a bug in :class:`preprocessing.Normalizer` with norm='max', + which was not taking the absolute value of the maximum values before + normalizing the vectors. :pr:`16632` by + :user:`Maura Pintor ` and :user:`Battista Biggio `. + +:mod:`sklearn.svm` +.................. + +- |API| :class:`svm.SVR` and :class:`svm.OneClassSVM` attributes, `probA_` and + `probB_`, are now deprecated as they were not useful. :pr:`15558` by + `Thomas Fan`_. + +- |Fix| Fix use of custom kernel not taking float entries such as string + kernels in :class:`svm.SVC` and :class:`svm.SVR`. Note that custom kennels + are now expected to validate their input where they previously received + valid numeric arrays. + :pr:`11296` by `Alexandre Gramfort`_ and :user:`Georgi Peev `. + +:mod:`sklearn.tree` +................... + +- |Fix| :func:`tree.plot_tree` `rotate` parameter was unused and has been + deprecated. + :pr:`15806` by :user:`Chiara Marmo `. + +- |Fix| Fix support of read-only float32 array input in ``predict``, + ``decision_path`` and ``predict_proba`` methods of + :class:`tree.DecisionTreeClassifier`, :class:`tree.ExtraTreeClassifier` and + :class:`ensemble.GradientBoostingClassifier` as well as ``predict`` method of + :class:`tree.DecisionTreeRegressor`, :class:`tree.ExtraTreeRegressor`, and + :class:`ensemble.GradientBoostingRegressor`. + :pr:`16331` by :user:`Alexandre Batisse `. + +:mod:`sklearn.utils` +.................... + +- |Enhancement| improve error message in :func:`utils.validation.column_or_1d`. + :pr:`15926` by :user:`Loïc Estève `. + +- |Enhancement| add warning in :func:`utils.validation.check_array` for + pandas sparse DataFrame. + :pr:`16021` by :user:`Rushabh Vasani `. + +:mod:`sklearn.cluster` +...................... + +- |Fix| :class:`cluster.AgglomerativeClustering` add specific error when + distance matrix is not square and `affinity=precomputed`. + :pr:`16257` by :user:`Simona Maggio `. + +Miscellaneous +............. + +- |API| Most estimators now expose a `n_features_in_` attribute. This + attribute is equal to the number of features passed to the `fit` method. + See `SLEP010 + `_ + for details. :pr:`16112` by `Nicolas Hug`_. diff --git a/examples/applications/svm_gui.py b/examples/applications/svm_gui.py index 46b7f7369a0fe..d085851422e18 100644 --- a/examples/applications/svm_gui.py +++ b/examples/applications/svm_gui.py @@ -22,9 +22,14 @@ import matplotlib matplotlib.use('TkAgg') - from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg -from matplotlib.backends.backend_tkagg import NavigationToolbar2TkAgg +try: + from matplotlib.backends.backend_tkagg import NavigationToolbar2Tk +except ImportError: + # NavigationToolbar2TkAgg was deprecated in matplotlib 2.2 + from matplotlib.backends.backend_tkagg import ( + NavigationToolbar2TkAgg as NavigationToolbar2Tk + ) from matplotlib.figure import Figure from matplotlib.contour import ContourSet @@ -144,11 +149,15 @@ def __init__(self, root, controller): ax.set_xlim((x_min, x_max)) ax.set_ylim((y_min, y_max)) canvas = FigureCanvasTkAgg(f, master=root) - canvas.show() + try: + canvas.draw() + except AttributeError: + # support for matplotlib (1.*) + canvas.show() canvas.get_tk_widget().pack(side=Tk.TOP, fill=Tk.BOTH, expand=1) canvas._tkcanvas.pack(side=Tk.TOP, fill=Tk.BOTH, expand=1) canvas.mpl_connect('button_press_event', self.onclick) - toolbar = NavigationToolbar2TkAgg(canvas, root) + toolbar = NavigationToolbar2Tk(canvas, root) toolbar.update() self.controllbar = ControllBar(root, controller) self.f = f diff --git a/examples/applications/wikipedia_principal_eigenvector.py b/examples/applications/wikipedia_principal_eigenvector.py index da4234936a911..097bab6c7d4d5 100644 --- a/examples/applications/wikipedia_principal_eigenvector.py +++ b/examples/applications/wikipedia_principal_eigenvector.py @@ -42,8 +42,6 @@ from scipy import sparse -from joblib import Memory - from sklearn.decomposition import randomized_svd from urllib.request import urlopen @@ -74,8 +72,6 @@ # ############################################################################# # Loading the redirect files -memory = Memory(cachedir=".") - def index(redirects, index_map, k): """Find the index of an article name after redirect resolution""" @@ -124,8 +120,6 @@ def get_redirects(redirects_filename): return redirects -# disabling joblib as the pickling of large dicts seems much too slow -#@memory.cache def get_adjacency_matrix(redirects_filename, page_links_filename, limit=None): """Extract the adjacency graph as a scipy sparse matrix diff --git a/examples/classification/plot_digits_classification.py b/examples/classification/plot_digits_classification.py index 21c4c81d15a62..6c7feb0f42065 100644 --- a/examples/classification/plot_digits_classification.py +++ b/examples/classification/plot_digits_classification.py @@ -31,12 +31,12 @@ # matplotlib.pyplot.imread. Note that each image must have the same size. For these # images, we know which digit they represent: it is given in the 'target' of # the dataset. +_, axes = plt.subplots(2, 4) images_and_labels = list(zip(digits.images, digits.target)) -for index, (image, label) in enumerate(images_and_labels[:4]): - plt.subplot(2, 4, index + 1) - plt.axis('off') - plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest') - plt.title('Training: %i' % label) +for ax, (image, label) in zip(axes[0, :], images_and_labels[:4]): + ax.set_axis_off() + ax.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest') + ax.set_title('Training: %i' % label) # To apply a classifier on this data, we need to flatten the image, to # turn the data in a (samples, feature) matrix: @@ -56,15 +56,16 @@ # Now predict the value of the digit on the second half: predicted = classifier.predict(X_test) +images_and_predictions = list(zip(digits.images[n_samples // 2:], predicted)) +for ax, (image, prediction) in zip(axes[1, :], images_and_predictions[:4]): + ax.set_axis_off() + ax.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest') + ax.set_title('Prediction: %i' % prediction) + print("Classification report for classifier %s:\n%s\n" % (classifier, metrics.classification_report(y_test, predicted))) -print("Confusion matrix:\n%s" % metrics.confusion_matrix(y_test, predicted)) - -images_and_predictions = list(zip(digits.images[n_samples // 2:], predicted)) -for index, (image, prediction) in enumerate(images_and_predictions[:4]): - plt.subplot(2, 4, index + 5) - plt.axis('off') - plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest') - plt.title('Prediction: %i' % prediction) +disp = metrics.plot_confusion_matrix(classifier, X_test, y_test) +disp.figure_.suptitle("Confusion Matrix") +print("Confusion matrix:\n%s" % disp.confusion_matrix) plt.show() diff --git a/examples/cluster/plot_agglomerative_clustering.py b/examples/cluster/plot_agglomerative_clustering.py index 79d3841cecfdc..b5fbce9e49b0d 100644 --- a/examples/cluster/plot_agglomerative_clustering.py +++ b/examples/cluster/plot_agglomerative_clustering.py @@ -71,7 +71,7 @@ plt.axis('equal') plt.axis('off') - plt.subplots_adjust(bottom=0, top=.89, wspace=0, + plt.subplots_adjust(bottom=0, top=.83, wspace=0, left=0, right=1) plt.suptitle('n_cluster=%i, connectivity=%r' % (n_clusters, connectivity is not None), size=17) diff --git a/examples/compose/plot_column_transformer_mixed_types.py b/examples/compose/plot_column_transformer_mixed_types.py index e21bc9634dda6..1c79c4bb1d607 100644 --- a/examples/compose/plot_column_transformer_mixed_types.py +++ b/examples/compose/plot_column_transformer_mixed_types.py @@ -3,21 +3,22 @@ Column Transformer with Mixed Types =================================== -This example illustrates how to apply different preprocessing and -feature extraction pipelines to different subsets of features, -using :class:`sklearn.compose.ColumnTransformer`. -This is particularly handy for the case of datasets that contain -heterogeneous data types, since we may want to scale the -numeric features and one-hot encode the categorical ones. - -In this example, the numeric data is standard-scaled after -mean-imputation, while the categorical data is one-hot -encoded after imputing missing values with a new category -(``'missing'``). - -Finally, the preprocessing pipeline is integrated in a -full prediction pipeline using :class:`sklearn.pipeline.Pipeline`, -together with a simple classification model. +This example illustrates how to apply different preprocessing and feature +extraction pipelines to different subsets of features, using +:class:`sklearn.compose.ColumnTransformer`. This is particularly handy for the +case of datasets that contain heterogeneous data types, since we may want to +scale the numeric features and one-hot encode the categorical ones. + +In this example, the numeric data is standard-scaled after mean-imputation, +while the categorical data is one-hot encoded after imputing missing values +with a new category (``'missing'``). + +In addition, we show two different ways to dispatch the columns to the +particular pre-processor: by column names and by column data types. + +Finally, the preprocessing pipeline is integrated in a full prediction pipeline +using :class:`sklearn.pipeline.Pipeline`, together with a simple classification +model. """ # Author: Pedro Morales @@ -43,16 +44,24 @@ # X = titanic.frame.drop('survived', axis=1) # y = titanic.frame['survived'] +############################################################################### +# Use ``ColumnTransformer`` by selecting column by names +############################################################################### # We will train our classifier with the following features: +# # Numeric Features: -# - age: float. -# - fare: float. +# +# * ``age``: float; +# * ``fare``: float. +# # Categorical Features: -# - embarked: categories encoded as strings {'C', 'S', 'Q'}. -# - sex: categories encoded as strings {'female', 'male'}. -# - pclass: ordinal integers {1, 2, 3}. - +# +# * ``embarked``: categories encoded as strings ``{'C', 'S', 'Q'}``; +# * ``sex``: categories encoded as strings ``{'female', 'male'}``; +# * ``pclass``: ordinal integers ``{1, 2, 3}``. +# # We create the preprocessing pipelines for both numeric and categorical data. + numeric_features = ['age', 'fare'] numeric_transformer = Pipeline(steps=[ ('imputer', SimpleImputer(strategy='median')), @@ -78,6 +87,50 @@ clf.fit(X_train, y_train) print("model score: %.3f" % clf.score(X_test, y_test)) +############################################################################### +# Use ``ColumnTransformer`` by selecting column by data types +############################################################################### +# When dealing with a cleaned dataset, the preprocessing can be automatic by +# using the data types of the column to decide whether to treat a column as a +# numerical or categorical feature. +# :func:`sklearn.compose.make_column_selector` gives this possibility. +# First, let's only select a subset of columns to simplify our +# example. + +subset_feature = ['embarked', 'sex', 'pclass', 'age', 'fare'] +X = X[subset_feature] + +############################################################################### +# Then, we introspect the information regarding each column data type. + +X.info() + +############################################################################### +# We can observe that the `embarked` and `sex` columns were tagged as +# `category` columns when loading the data with ``fetch_openml``. Therefore, we +# can use this information to dispatch the categorical columns to the +# ``categorical_transformer`` and the remaining columns to the +# ``numerical_transformer``. + +############################################################################### +# .. note:: In practice, you will have to handle yourself the column data type. +# If you want some columns to be considered as `category`, you will have to +# convert them into categorical columns. If you are using pandas, you can +# refer to their documentation regarding `Categorical data +# `_. + +from sklearn.compose import make_column_selector as selector + +preprocessor = ColumnTransformer(transformers=[ + ('num', numeric_transformer, selector(dtype_exclude="category")), + ('cat', categorical_transformer, selector(dtype_include="category")) +]) + +# Reproduce the identical fit/score process +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) + +clf.fit(X_train, y_train) +print("model score: %.3f" % clf.score(X_test, y_test)) ############################################################################### # Using the prediction pipeline in a grid search @@ -89,7 +142,6 @@ # and the regularization parameter of the logistic regression using # :class:`sklearn.model_selection.GridSearchCV`. - param_grid = { 'preprocessor__num__imputer__strategy': ['mean', 'median'], 'classifier__C': [0.1, 1.0, 10, 100], diff --git a/examples/ensemble/plot_forest_importances.py b/examples/ensemble/plot_forest_importances.py index b53c27331a9f4..4a6866b0b5b03 100644 --- a/examples/ensemble/plot_forest_importances.py +++ b/examples/ensemble/plot_forest_importances.py @@ -4,11 +4,17 @@ ========================================= This examples shows the use of forests of trees to evaluate the importance of -features on an artificial classification task. The red bars are the feature -importances of the forest, along with their inter-trees variability. +features on an artificial classification task. The red bars are +the impurity-based feature importances of the forest, +along with their inter-trees variability. As expected, the plot suggests that 3 features are informative, while the remaining are not. + +Warning: impurity-based feature importances can be misleading for high +cardinality features (many unique values). See +:func:`sklearn.inspection.permutation_importance` as an alternative. + """ print(__doc__) @@ -28,7 +34,7 @@ random_state=0, shuffle=False) -# Build a forest and compute the feature importances +# Build a forest and compute the impurity-based feature importances forest = ExtraTreesClassifier(n_estimators=250, random_state=0) @@ -44,11 +50,11 @@ for f in range(X.shape[1]): print("%d. feature %d (%f)" % (f + 1, indices[f], importances[indices[f]])) -# Plot the feature importances of the forest +# Plot the impurity-based feature importances of the forest plt.figure() plt.title("Feature importances") plt.bar(range(X.shape[1]), importances[indices], - color="r", yerr=std[indices], align="center") + color="r", yerr=std[indices], align="center") plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.show() diff --git a/examples/ensemble/plot_forest_importances_faces.py b/examples/ensemble/plot_forest_importances_faces.py index f0649845a2867..6cea84ca4744c 100644 --- a/examples/ensemble/plot_forest_importances_faces.py +++ b/examples/ensemble/plot_forest_importances_faces.py @@ -3,9 +3,9 @@ Pixel importances with a parallel forest of trees ================================================= -This example shows the use of forests of trees to evaluate the importance -of the pixels in an image classification task (faces). The hotter the pixel, -the more important. +This example shows the use of forests of trees to evaluate the impurity-based +importance of the pixels in an image classification task (faces). +The hotter the pixel, the more important. The code below also illustrates how the construction and the computation of the predictions can be parallelized within multiple jobs. diff --git a/examples/ensemble/plot_gradient_boosting_regression.py b/examples/ensemble/plot_gradient_boosting_regression.py index 9285f8dae0eea..bab88d71844d9 100644 --- a/examples/ensemble/plot_gradient_boosting_regression.py +++ b/examples/ensemble/plot_gradient_boosting_regression.py @@ -62,7 +62,12 @@ plt.ylabel('Deviance') # ############################################################################# -# Plot feature importance +# Plot impurity-based feature importance +# +# Warning: impurity-based feature importances can be misleading for +# high cardinality features (many unique values). See +# :func:`sklearn.inspection.permutation_importance` as an alternative. + feature_importance = clf.feature_importances_ # make importances relative to max importance feature_importance = 100.0 * (feature_importance / feature_importance.max()) diff --git a/examples/ensemble/plot_stack_predictors.py b/examples/ensemble/plot_stack_predictors.py index 0a3e12646b427..3d2ff4a38ddd7 100644 --- a/examples/ensemble/plot_stack_predictors.py +++ b/examples/ensemble/plot_stack_predictors.py @@ -3,6 +3,8 @@ Combine predictors using stacking ================================= +.. currentmodule:: sklearn + Stacking refers to a method to blend estimators. In this strategy, some estimators are individually fitted on some training data while a final estimator is trained using the stacked predictions of these base estimators. @@ -16,42 +18,128 @@ print(__doc__) # Authors: Guillaume Lemaitre +# Maria Telenczuk # License: BSD 3 clause + ############################################################################### -# The function ``plot_regression_results`` is used to plot the predicted and -# true targets. +# Download the dataset +############################################################################### +# +# We will use `Ames Housing`_ dataset which was first compiled by Dean De Cock +# and became better known after it was used in Kaggle challenge. It is a set +# of 1460 residential homes in Ames, Iowa, each described by 80 features. We +# will use it to predict the final logarithmic price of the houses. In this +# example we will use only 20 most interesting features chosen using +# GradientBoostingRegressor() and limit number of entries (here we won't go +# into the details on how to select the most interesting features). +# +# The Ames housing dataset is not shipped with scikit-learn and therefore we +# will fetch it from `OpenML`_. +# +# .. _`Ames Housing`: http://jse.amstat.org/v19n3/decock.pdf +# .. _`OpenML`: https://www.openml.org/d/42165 -import matplotlib.pyplot as plt +import numpy as np +from sklearn.datasets import fetch_openml +from sklearn.utils import shuffle -def plot_regression_results(ax, y_true, y_pred, title, scores, elapsed_time): - """Scatter plot of the predicted vs true targets.""" - ax.plot([y_true.min(), y_true.max()], - [y_true.min(), y_true.max()], - '--r', linewidth=2) - ax.scatter(y_true, y_pred, alpha=0.2) - ax.spines['top'].set_visible(False) - ax.spines['right'].set_visible(False) - ax.get_xaxis().tick_bottom() - ax.get_yaxis().tick_left() - ax.spines['left'].set_position(('outward', 10)) - ax.spines['bottom'].set_position(('outward', 10)) - ax.set_xlim([y_true.min(), y_true.max()]) - ax.set_ylim([y_true.min(), y_true.max()]) - ax.set_xlabel('Measured') - ax.set_ylabel('Predicted') - extra = plt.Rectangle((0, 0), 0, 0, fc="w", fill=False, - edgecolor='none', linewidth=0) - ax.legend([extra], [scores], loc='upper left') - title = title + '\n Evaluation in {:.2f} seconds'.format(elapsed_time) - ax.set_title(title) +def load_ames_housing(): + df = fetch_openml(name="house_prices", as_frame=True) + X = df.data + y = df.target + + features = ['YrSold', 'HeatingQC', 'Street', 'YearRemodAdd', 'Heating', + 'MasVnrType', 'BsmtUnfSF', 'Foundation', 'MasVnrArea', + 'MSSubClass', 'ExterQual', 'Condition2', 'GarageCars', + 'GarageType', 'OverallQual', 'TotalBsmtSF', 'BsmtFinSF1', + 'HouseStyle', 'MiscFeature', 'MoSold'] + + X = X[features] + X, y = shuffle(X, y, random_state=0) + + X = X[:600] + y = y[:600] + return X, np.log(y) + + +X, y = load_ames_housing() + + +############################################################################### +# Make pipeline to preprocess the data +############################################################################### +# +# Before we can use Ames dataset we still need to do some preprocessing. +# First, the dataset has many missing values. To impute them, we will exchange +# categorical missing values with the new category 'missing' while the +# numerical missing values with the 'mean' of the column. We will also encode +# the categories with either :class:`sklearn.preprocessing.OneHotEncoder +# ` or +# :class:`sklearn.preprocessing.OrdinalEncoder +# ` depending for which type of model we +# will use them (linear or non-linear model). To falicitate this preprocessing +# we will make two pipelines. +# You can skip this section if your data is ready to use and does +# not need preprocessing + + +from sklearn.compose import make_column_transformer +from sklearn.impute import SimpleImputer +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import OneHotEncoder +from sklearn.preprocessing import OrdinalEncoder +from sklearn.preprocessing import StandardScaler + + +cat_cols = X.columns[X.dtypes == 'O'] +num_cols = X.columns[X.dtypes == 'float64'] + +categories = [ + X[column].unique() for column in X[cat_cols]] + +for cat in categories: + cat[cat == None] = 'missing' # noqa + +cat_proc_nlin = make_pipeline( + SimpleImputer(missing_values=None, strategy='constant', + fill_value='missing'), + OrdinalEncoder(categories=categories) + ) + +num_proc_nlin = make_pipeline(SimpleImputer(strategy='mean')) + +cat_proc_lin = make_pipeline( + SimpleImputer(missing_values=None, + strategy='constant', + fill_value='missing'), + OneHotEncoder(categories=categories) +) + +num_proc_lin = make_pipeline( + SimpleImputer(strategy='mean'), + StandardScaler() +) + +# transformation to use for non-linear estimators +processor_nlin = make_column_transformer( + (cat_proc_nlin, cat_cols), + (num_proc_nlin, num_cols), + remainder='passthrough') + +# transformation to use for linear estimators +processor_lin = make_column_transformer( + (cat_proc_lin, cat_cols), + (num_proc_lin, num_cols), + remainder='passthrough') ############################################################################### # Stack of predictors on a single data set ############################################################################### +# # It is sometimes tedious to find the model which will best perform on a given # dataset. Stacking provide an alternative by combining the outputs of several # learners, without the need to choose a model specifically. The performance of @@ -60,35 +148,79 @@ def plot_regression_results(ax, y_true, y_pred, title, scores, elapsed_time): # # Here, we combine 3 learners (linear and non-linear) and use a ridge regressor # to combine their outputs together. +# +# Note: although we will make new pipelines with the processors which we wrote +# in the previous section for the 3 learners, the final estimator RidgeCV() +# does not need preprocessing of the data as it will be fed with the already +# preprocessed output from the 3 learners. + -from sklearn.ensemble import StackingRegressor -from sklearn.ensemble import RandomForestRegressor from sklearn.experimental import enable_hist_gradient_boosting # noqa from sklearn.ensemble import HistGradientBoostingRegressor +from sklearn.ensemble import RandomForestRegressor +from sklearn.ensemble import StackingRegressor from sklearn.linear_model import LassoCV from sklearn.linear_model import RidgeCV -estimators = [ - ('Random Forest', RandomForestRegressor(random_state=42)), - ('Lasso', LassoCV()), - ('Gradient Boosting', HistGradientBoostingRegressor(random_state=0)) -] -stacking_regressor = StackingRegressor( - estimators=estimators, final_estimator=RidgeCV() -) +lasso_pipeline = make_pipeline(processor_lin, + LassoCV()) + +rf_pipeline = make_pipeline(processor_nlin, + RandomForestRegressor(random_state=42)) +gradient_pipeline = make_pipeline( + processor_nlin, + HistGradientBoostingRegressor(random_state=0)) + +estimators = [('Random Forest', rf_pipeline), + ('Lasso', lasso_pipeline), + ('Gradient Boosting', gradient_pipeline)] + +stacking_regressor = StackingRegressor(estimators=estimators, + final_estimator=RidgeCV()) + + +############################################################################### +# Measure and plot the results ############################################################################### -# We used the Boston data set (prediction of house prices). We check the -# performance of each individual predictor as well as the stack of the +# +# Now we can use Ames Housing dataset to make the predictions. We check the +# performance of each individual predictor as well as of the stack of the # regressors. +# +# The function ``plot_regression_results`` is used to plot the predicted and +# true targets. + import time -import numpy as np -from sklearn.datasets import load_boston +import matplotlib.pyplot as plt from sklearn.model_selection import cross_validate, cross_val_predict -X, y = load_boston(return_X_y=True) + +def plot_regression_results(ax, y_true, y_pred, title, scores, elapsed_time): + """Scatter plot of the predicted vs true targets.""" + ax.plot([y_true.min(), y_true.max()], + [y_true.min(), y_true.max()], + '--r', linewidth=2) + ax.scatter(y_true, y_pred, alpha=0.2) + + ax.spines['top'].set_visible(False) + ax.spines['right'].set_visible(False) + ax.get_xaxis().tick_bottom() + ax.get_yaxis().tick_left() + ax.spines['left'].set_position(('outward', 10)) + ax.spines['bottom'].set_position(('outward', 10)) + ax.set_xlim([y_true.min(), y_true.max()]) + ax.set_ylim([y_true.min(), y_true.max()]) + ax.set_xlabel('Measured') + ax.set_ylabel('Predicted') + extra = plt.Rectangle((0, 0), 0, 0, fc="w", fill=False, + edgecolor='none', linewidth=0) + ax.legend([extra], [scores], loc='upper left') + title = title + '\n Evaluation in {:.2f} seconds'.format(elapsed_time) + ax.set_title(title) + fig, axs = plt.subplots(2, 2, figsize=(9, 7)) axs = np.ravel(axs) @@ -102,6 +234,7 @@ def plot_regression_results(ax, y_true, y_pred, title, scores, elapsed_time): elapsed_time = time.time() - start_time y_pred = cross_val_predict(est, X, y, n_jobs=-1, verbose=0) + plot_regression_results( ax, y, y_pred, name, diff --git a/examples/ensemble/plot_voting_regressor.py b/examples/ensemble/plot_voting_regressor.py index 973c9395505de..6fd629bb9c083 100644 --- a/examples/ensemble/plot_voting_regressor.py +++ b/examples/ensemble/plot_voting_regressor.py @@ -5,15 +5,21 @@ .. currentmodule:: sklearn -Plot individual and averaged regression predictions for Boston dataset. - -First, three exemplary regressors are initialized -(:class:`~ensemble.GradientBoostingRegressor`, +A voting regressor is an ensemble meta-estimator that fits base regressors each +on the whole dataset. It, then, averages the individual predictions to form a +final prediction. +We will use three different regressors to predict the data: +:class:`~ensemble.GradientBoostingRegressor`, :class:`~ensemble.RandomForestRegressor`, and -:class:`~linear_model.LinearRegression`) and used to initialize a +:class:`~linear_model.LinearRegression`). +Then, using them we will make voting regressor :class:`~ensemble.VotingRegressor`. -The red starred dots are the averaged predictions. +Finally, we will plot all of them for comparison. + +We will work with the diabetes dataset which consists of the 10 features +collected from a cohort of diabetes patients. The target is the disease +progression after one year from the baseline. """ print(__doc__) @@ -26,30 +32,60 @@ from sklearn.linear_model import LinearRegression from sklearn.ensemble import VotingRegressor -# Loading some example data -X, y = datasets.load_boston(return_X_y=True) - +############################################################################## # Training classifiers -reg1 = GradientBoostingRegressor(random_state=1, n_estimators=10) -reg2 = RandomForestRegressor(random_state=1, n_estimators=10) +# -------------------------------- +# +# First, we are going to load diabetes dataset and initiate gradient boosting +# regressor, random forest regressor and linear regression. Next, we are going +# to use each of them to build the voting regressor: + +X, y = datasets.load_diabetes(return_X_y=True) + +# Train classifiers +reg1 = GradientBoostingRegressor(random_state=1) +reg2 = RandomForestRegressor(random_state=1) reg3 = LinearRegression() -ereg = VotingRegressor([('gb', reg1), ('rf', reg2), ('lr', reg3)]) + reg1.fit(X, y) reg2.fit(X, y) reg3.fit(X, y) + +ereg = VotingRegressor([('gb', reg1), ('rf', reg2), ('lr', reg3)]) ereg.fit(X, y) +############################################################################## +# Making predictions +# -------------------------------- +# +# Now we will use each of the regressors to make 20 first predictions about the +# diabetes dataset. + xt = X[:20] +pred1 = reg1.predict(xt) +pred2 = reg2.predict(xt) +pred3 = reg3.predict(xt) +pred4 = ereg.predict(xt) + +############################################################################## +# Plot the results +# -------------------------------- +# +# Finally, we will visualize the 20 predictions. The red stars show the average +# prediction + plt.figure() -plt.plot(reg1.predict(xt), 'gd', label='GradientBoostingRegressor') -plt.plot(reg2.predict(xt), 'b^', label='RandomForestRegressor') -plt.plot(reg3.predict(xt), 'ys', label='LinearRegression') -plt.plot(ereg.predict(xt), 'r*', label='VotingRegressor') +plt.plot(pred1, 'gd', label='GradientBoostingRegressor') +plt.plot(pred2, 'b^', label='RandomForestRegressor') +plt.plot(pred3, 'ys', label='LinearRegression') +plt.plot(pred4, 'r*', ms=10, label='VotingRegressor') + plt.tick_params(axis='x', which='both', bottom=False, top=False, labelbottom=False) plt.ylabel('predicted') plt.xlabel('training samples') plt.legend(loc="best") -plt.title('Comparison of individual predictions with averaged') +plt.title('Regressor predictions and their average') + plt.show() diff --git a/examples/feature_selection/plot_select_from_model_boston.py b/examples/feature_selection/plot_select_from_model_boston.py deleted file mode 100644 index 8e524909e8c9a..0000000000000 --- a/examples/feature_selection/plot_select_from_model_boston.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -=================================================== -Feature selection using SelectFromModel and LassoCV -=================================================== - -Use SelectFromModel meta-transformer along with Lasso to select the best -couple of features from the Boston dataset. -""" -# Author: Manoj Kumar -# License: BSD 3 clause - -print(__doc__) - -import matplotlib.pyplot as plt -import numpy as np - -from sklearn.datasets import load_boston -from sklearn.feature_selection import SelectFromModel -from sklearn.linear_model import LassoCV - -# Load the boston dataset. -X, y = load_boston(return_X_y=True) - -# We use the base estimator LassoCV since the L1 norm promotes sparsity of features. -clf = LassoCV() - -# Set a minimum threshold of 0.25 -sfm = SelectFromModel(clf, threshold=0.25) -sfm.fit(X, y) -n_features = sfm.transform(X).shape[1] - -# Reset the threshold till the number of features equals two. -# Note that the attribute can be set directly instead of repeatedly -# fitting the metatransformer. -while n_features > 2: - sfm.threshold += 0.1 - X_transform = sfm.transform(X) - n_features = X_transform.shape[1] - -# Plot the selected two features from X. -plt.title( - "Features selected from Boston using SelectFromModel with " - "threshold %0.3f." % sfm.threshold) -feature1 = X_transform[:, 0] -feature2 = X_transform[:, 1] -plt.plot(feature1, feature2, 'r.') -plt.xlabel("Feature number 1") -plt.ylabel("Feature number 2") -plt.ylim([np.min(feature2), np.max(feature2)]) -plt.show() diff --git a/examples/feature_selection/plot_select_from_model_diabetes.py b/examples/feature_selection/plot_select_from_model_diabetes.py new file mode 100644 index 0000000000000..0a7e46448a22c --- /dev/null +++ b/examples/feature_selection/plot_select_from_model_diabetes.py @@ -0,0 +1,100 @@ +""" +=================================================== +Feature selection using SelectFromModel and LassoCV +=================================================== + +Use SelectFromModel meta-transformer along with Lasso to select the best +couple of features from the diabetes dataset. + +Since the L1 norm promotes sparsity of features we might be interested in +selecting only a subset of the most interesting features from the dataset. This +example shows how to select two the most interesting features from the diabetes +dataset. + +Diabetes dataset consists of 10 variables (features) collected from 442 +diabetes patients. This example shows how to use SelectFromModel and LassoCv to +find the best two features predicting disease progression after one year from +the baseline. + +Authors: `Manoj Kumar `_, +`Maria Telenczuk `_ + +License: BSD 3 clause +""" + +print(__doc__) + +import matplotlib.pyplot as plt +import numpy as np + +from sklearn.datasets import load_diabetes +from sklearn.feature_selection import SelectFromModel +from sklearn.linear_model import LassoCV + +############################################################################## +# Load the data +# --------------------------------------------------------- +# +# First, let's load the diabetes dataset which is available from within +# sklearn. Then, we will look what features are collected for the diabates +# patients: + +diabetes = load_diabetes() + +X = diabetes.data +y = diabetes.target + +feature_names = diabetes.feature_names +print(feature_names) + +############################################################################## +# Find importance of the features +# --------------------------------------------------------- +# +# To decide on the importance of the features we are going to use LassoCV +# estimator. The features with the highest absolute `coef_` value are +# considered the most important + +clf = LassoCV().fit(X, y) +importance = np.abs(clf.coef_) +print(importance) + +############################################################################## +# Select from the model features with the higest score +# --------------------------------------------------------- +# +# Now we want to select the two features which are the most important. +# SelectFromModel() allows for setting the threshold. Only the features with +# the `coef_` higher than the threshold will remain. Here, we want to set the +# threshold slightly above the third highest `coef_` calculated by LassoCV() +# from our data. + +idx_third = importance.argsort()[-3] +threshold = importance[idx_third] + 0.01 + +idx_features = (-importance).argsort()[:2] +name_features = np.array(feature_names)[idx_features] +print('Selected features: {}'.format(name_features)) + +sfm = SelectFromModel(clf, threshold=threshold) +sfm.fit(X, y) +X_transform = sfm.transform(X) + +n_features = sfm.transform(X).shape[1] + +############################################################################## +# Plot the two most important features +# --------------------------------------------------------- +# +# Finally we will plot the selected two features from the data. + +plt.title( + "Features from diabets using SelectFromModel with " + "threshold %0.3f." % sfm.threshold) +feature1 = X_transform[:, 0] +feature2 = X_transform[:, 1] +plt.plot(feature1, feature2, 'r.') +plt.xlabel("First feature: {}".format(name_features[0])) +plt.ylabel("Second feature: {}".format(name_features[1])) +plt.ylim([np.min(feature2), np.max(feature2)]) +plt.show() diff --git a/examples/gaussian_process/plot_gpr_on_structured_data.py b/examples/gaussian_process/plot_gpr_on_structured_data.py new file mode 100644 index 0000000000000..64a84ab38647a --- /dev/null +++ b/examples/gaussian_process/plot_gpr_on_structured_data.py @@ -0,0 +1,174 @@ +""" +========================================================================== +Gaussian processes on discrete data structures +========================================================================== + +This example illustrates the use of Gaussian processes for regression and +classification tasks on data that are not in fixed-length feature vector form. +This is achieved through the use of kernel functions that operates directly +on discrete structures such as variable-length sequences, trees, and graphs. + +Specifically, here the input variables are some gene sequences stored as +variable-length strings consisting of letters 'A', 'T', 'C', and 'G', +while the output variables are floating point numbers and True/False labels +in the regression and classification tasks, respectively. + +A kernel between the gene sequences is defined using R-convolution [1]_ by +integrating a binary letter-wise kernel over all pairs of letters among a pair +of strings. + +This example will generate three figures. + +In the first figure, we visualize the value of the kernel, i.e. the similarity +of the sequences, using a colormap. Brighter color here indicates higher +similarity. + +In the second figure, we show some regression result on a dataset of 6 +sequences. Here we use the 1st, 2nd, 4th, and 5th sequences as the training set +to make predictions on the 3rd and 6th sequences. + +In the third figure, we demonstrate a classification model by training on 6 +sequences and make predictions on another 5 sequences. The ground truth here is +simply whether there is at least one 'A' in the sequence. Here the model makes +four correct classifications and fails on one. + +.. [1] Haussler, D. (1999). Convolution kernels on discrete structures + (Vol. 646). Technical report, Department of Computer Science, University + of California at Santa Cruz. +""" +print(__doc__) + +import numpy as np +import matplotlib.pyplot as plt +from sklearn.gaussian_process.kernels import Kernel, Hyperparameter +from sklearn.gaussian_process.kernels import GenericKernelMixin +from sklearn.gaussian_process import GaussianProcessRegressor +from sklearn.gaussian_process import GaussianProcessClassifier +from sklearn.base import clone + + +class SequenceKernel(GenericKernelMixin, Kernel): + ''' + A minimal (but valid) convolutional kernel for sequences of variable + lengths.''' + def __init__(self, + baseline_similarity=0.5, + baseline_similarity_bounds=(1e-5, 1)): + self.baseline_similarity = baseline_similarity + self.baseline_similarity_bounds = baseline_similarity_bounds + + @property + def hyperparameter_baseline_similarity(self): + return Hyperparameter("baseline_similarity", + "numeric", + self.baseline_similarity_bounds) + + def _f(self, s1, s2): + ''' + kernel value between a pair of sequences + ''' + return sum([1.0 if c1 == c2 else self.baseline_similarity + for c1 in s1 + for c2 in s2]) + + def _g(self, s1, s2): + ''' + kernel derivative between a pair of sequences + ''' + return sum([0.0 if c1 == c2 else 1.0 + for c1 in s1 + for c2 in s2]) + + def __call__(self, X, Y=None, eval_gradient=False): + if Y is None: + Y = X + + if eval_gradient: + return (np.array([[self._f(x, y) for y in Y] for x in X]), + np.array([[[self._g(x, y)] for y in Y] for x in X])) + else: + return np.array([[self._f(x, y) for y in Y] for x in X]) + + def diag(self, X): + return np.array([self._f(x, x) for x in X]) + + def is_stationary(self): + return False + + def clone_with_theta(self, theta): + cloned = clone(self) + cloned.theta = theta + return cloned + + +kernel = SequenceKernel() + +''' +Sequence similarity matrix under the kernel +=========================================== +''' + +X = np.array(['AGCT', 'AGC', 'AACT', 'TAA', 'AAA', 'GAACA']) + +K = kernel(X) +D = kernel.diag(X) + +plt.figure(figsize=(8, 5)) +plt.imshow(np.diag(D**-0.5).dot(K).dot(np.diag(D**-0.5))) +plt.xticks(np.arange(len(X)), X) +plt.yticks(np.arange(len(X)), X) +plt.title('Sequence similarity under the kernel') + +''' +Regression +========== +''' + +X = np.array(['AGCT', 'AGC', 'AACT', 'TAA', 'AAA', 'GAACA']) +Y = np.array([1.0, 1.0, 2.0, 2.0, 3.0, 3.0]) + +training_idx = [0, 1, 3, 4] +gp = GaussianProcessRegressor(kernel=kernel) +gp.fit(X[training_idx], Y[training_idx]) + +plt.figure(figsize=(8, 5)) +plt.bar(np.arange(len(X)), gp.predict(X), color='b', label='prediction') +plt.bar(training_idx, Y[training_idx], width=0.2, color='r', + alpha=1, label='training') +plt.xticks(np.arange(len(X)), X) +plt.title('Regression on sequences') +plt.legend() + +''' +Classification +============== +''' + +X_train = np.array(['AGCT', 'CGA', 'TAAC', 'TCG', 'CTTT', 'TGCT']) +# whether there are 'A's in the sequence +Y_train = np.array([True, True, True, False, False, False]) + +gp = GaussianProcessClassifier(kernel) +gp.fit(X_train, Y_train) + +X_test = ['AAA', 'ATAG', 'CTC', 'CT', 'C'] +Y_test = [True, True, False, False, False] + +plt.figure(figsize=(8, 5)) +plt.scatter(np.arange(len(X_train)), [1.0 if c else -1.0 for c in Y_train], + s=100, marker='o', edgecolor='none', facecolor=(1, 0.75, 0), + label='training') +plt.scatter(len(X_train) + np.arange(len(X_test)), + [1.0 if c else -1.0 for c in Y_test], + s=100, marker='o', edgecolor='none', facecolor='r', label='truth') +plt.scatter(len(X_train) + np.arange(len(X_test)), + [1.0 if c else -1.0 for c in gp.predict(X_test)], + s=100, marker='x', edgecolor=(0, 1.0, 0.3), linewidth=2, + label='prediction') +plt.xticks(np.arange(len(X_train) + len(X_test)), + np.concatenate((X_train, X_test))) +plt.yticks([-1, 1], [False, True]) +plt.title('Classification on sequences') +plt.legend() + +plt.show() diff --git a/examples/impute/plot_iterative_imputer_variants_comparison.py b/examples/impute/plot_iterative_imputer_variants_comparison.py index 06fab08c381f2..90e8e4cad1a9b 100644 --- a/examples/impute/plot_iterative_imputer_variants_comparison.py +++ b/examples/impute/plot_iterative_imputer_variants_comparison.py @@ -127,6 +127,6 @@ ax.set_title('California Housing Regression with Different Imputation Methods') ax.set_xlabel('MSE (smaller is better)') ax.set_yticks(np.arange(means.shape[0])) -ax.set_yticklabels([" w/ ".join(label) for label in means.index.get_values()]) +ax.set_yticklabels([" w/ ".join(label) for label in means.index.tolist()]) plt.tight_layout(pad=1) plt.show() diff --git a/examples/inspection/plot_partial_dependence.py b/examples/inspection/plot_partial_dependence.py index 526ace208e30f..d74c6363dec06 100644 --- a/examples/inspection/plot_partial_dependence.py +++ b/examples/inspection/plot_partial_dependence.py @@ -14,7 +14,7 @@ :class:`~sklearn.ensemble.HistGradientBoostingRegressor` trained on the California housing dataset. The example is taken from [1]_. -The plots show four 1-way and two 1-way partial dependence plots (ommitted for +The plots show four 1-way and two 1-way partial dependence plots (omitted for :class:`~sklearn.neural_network.MLPRegressor` due to computation time). The target variables for the one-way PDP are: median income (`MedInc`), average occupants per household (`AvgOccup`), median house age (`HouseAge`), and diff --git a/examples/inspection/plot_permutation_importance_multicollinear.py b/examples/inspection/plot_permutation_importance_multicollinear.py index eb8b591bb4f2b..5f832ffbd4228 100644 --- a/examples/inspection/plot_permutation_importance_multicollinear.py +++ b/examples/inspection/plot_permutation_importance_multicollinear.py @@ -60,11 +60,11 @@ fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 8)) ax1.barh(tree_indices, clf.feature_importances_[tree_importance_sorted_idx], height=0.7) -ax1.set_yticklabels(data.feature_names) +ax1.set_yticklabels(data.feature_names[tree_importance_sorted_idx]) ax1.set_yticks(tree_indices) ax1.set_ylim((0, len(clf.feature_importances_))) ax2.boxplot(result.importances[perm_sorted_idx].T, vert=False, - labels=data.feature_names) + labels=data.feature_names[perm_sorted_idx]) fig.tight_layout() plt.show() diff --git a/examples/linear_model/plot_logistic_path.py b/examples/linear_model/plot_logistic_path.py index 79b5522575eb0..7aead065f3445 100644 --- a/examples/linear_model/plot_logistic_path.py +++ b/examples/linear_model/plot_logistic_path.py @@ -14,7 +14,7 @@ coefficients are exactly 0. When regularization gets progressively looser, coefficients can get non-zero values one after the other. -Here we choose the SAGA solver because it can efficiently optimize for the +Here we choose the liblinear solver because it can efficiently optimize for the Logistic Regression loss with a non-smooth, sparsity inducing l1 penalty. Also note that we set a low value for the tolerance to make sure that the model @@ -55,9 +55,10 @@ print("Computing regularization path ...") start = time() -clf = linear_model.LogisticRegression(penalty='l1', solver='saga', +clf = linear_model.LogisticRegression(penalty='l1', solver='liblinear', tol=1e-6, max_iter=int(1e6), - warm_start=True) + warm_start=True, + intercept_scaling=10000.) coefs_ = [] for c in cs: clf.set_params(C=c) diff --git a/examples/linear_model/plot_poisson_regression_non_normal_loss.py b/examples/linear_model/plot_poisson_regression_non_normal_loss.py new file mode 100644 index 0000000000000..4b0386edfcdf6 --- /dev/null +++ b/examples/linear_model/plot_poisson_regression_non_normal_loss.py @@ -0,0 +1,455 @@ +""" +====================================== +Poisson regression and non-normal loss +====================================== + +This example illustrates the use of log-linear Poisson regression +on the `French Motor Third-Party Liability Claims dataset +`_ from [1]_ and compares +it with models learned with least squared error. In this dataset, each sample +corresponds to an insurance policy, i.e. a contract within an insurance +company and an individual (policiholder). Available features include driver +age, vehicle age, vehicle power, etc. + +A few definitions: a *claim* is the request made by a policyholder to the +insurer to compensate for a loss covered by the insurance. The *exposure* is +the duration of the insurance coverage of a given policy, in years. + +Our goal is to predict the expected number of insurance claims (or frequency) +following car accidents for a policyholder given the historical data over a +population of policyholders. + +.. [1] A. Noll, R. Salzmann and M.V. Wuthrich, Case Study: French Motor + Third-Party Liability Claims (November 8, 2018). + `doi:10.2139/ssrn.3164764 `_ + +""" +print(__doc__) + +# Authors: Christian Lorentzen +# Roman Yurchak +# License: BSD 3 clause +import warnings + +import numpy as np +import matplotlib.pyplot as plt +import pandas as pd + +from sklearn.datasets import fetch_openml +from sklearn.dummy import DummyRegressor +from sklearn.compose import ColumnTransformer +from sklearn.linear_model import Ridge, PoissonRegressor +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import FunctionTransformer, OneHotEncoder +from sklearn.preprocessing import OrdinalEncoder +from sklearn.preprocessing import StandardScaler, KBinsDiscretizer +from sklearn.ensemble import RandomForestRegressor +from sklearn.utils import gen_even_slices +from sklearn.metrics import auc + +from sklearn.metrics import mean_squared_error, mean_absolute_error +from sklearn.metrics import mean_poisson_deviance + + +def load_mtpl2(n_samples=100000): + """Fetch the French Motor Third-Party Liability Claims dataset. + + Parameters + ---------- + n_samples: int or None, default=100000 + Number of samples to select (for faster run time). If None, the full + dataset with 678013 samples is returned. + """ + + # freMTPL2freq dataset from https://www.openml.org/d/41214 + df = fetch_openml(data_id=41214, as_frame=True)['data'] + + # unquote string fields + for column_name in df.columns[df.dtypes.values == np.object]: + df[column_name] = df[column_name].str.strip("'") + if n_samples is not None: + return df.iloc[:n_samples] + return df + + +############################################################################## +# Let's load the motor claim dataset. We ignore the severity data for this +# study for the sake of simplicitly. +# +# We also subsample the data for the sake of computational cost and running +# time. Using the full dataset would lead to similar conclusions. + +df = load_mtpl2(n_samples=300000) + +# Correct for unreasonable observations (that might be data error) +df["Exposure"] = df["Exposure"].clip(upper=1) + +############################################################################## +# The remaining columns can be used to predict the frequency of claim events. +# Those columns are very heterogeneous with a mix of categorical and numeric +# variables with different scales, possibly very unevenly distributed. +# +# In order to fit linear models with those predictors it is therefore +# necessary to perform standard feature transformations as follows: + +log_scale_transformer = make_pipeline( + FunctionTransformer(np.log, validate=False), + StandardScaler() +) + +linear_model_preprocessor = ColumnTransformer( + [ + ("passthrough_numeric", "passthrough", + ["BonusMalus"]), + ("binned_numeric", KBinsDiscretizer(n_bins=10), + ["VehAge", "DrivAge"]), + ("log_scaled_numeric", log_scale_transformer, + ["Density"]), + ("onehot_categorical", OneHotEncoder(), + ["VehBrand", "VehPower", "VehGas", "Region", "Area"]), + ], + remainder="drop", +) + +############################################################################## +# The number of claims (``ClaimNb``) is a positive integer that can be modeled +# as a Poisson distribution. It is then assumed to be the number of discrete +# events occurring with a constant rate in a given time interval +# (``Exposure``, in units of years). Here we model the frequency +# ``y = ClaimNb / Exposure``, which is still a (scaled) Poisson distribution, +# and use ``Exposure`` as ``sample_weight``. + +df["Frequency"] = df["ClaimNb"] / df["Exposure"] + +print( + pd.cut(df["Frequency"], [-1e-6, 1e-6, 1, 2, 3, 4, 5]).value_counts() +) + +print("Average Frequency = {}" + .format(np.average(df["Frequency"], weights=df["Exposure"]))) + +print("Percentage of zero claims = {0:%}" + .format(df.loc[df["ClaimNb"] == 0, "Exposure"].sum() / + df["Exposure"].sum())) + +############################################################################## +# It is worth noting that 92 % of policyholders have zero claims, and if we +# were to convert this problem into a binary classification task, it would be +# significantly imbalanced. +# +# To evaluate the pertinence of the used metrics, we will consider as a +# baseline a "dummy" estimator that constantly predicts the mean frequency of +# the training sample. + +df_train, df_test = train_test_split(df, random_state=0) + +dummy = make_pipeline( + linear_model_preprocessor, + DummyRegressor(strategy='mean') +) +dummy.fit(df_train, df_train["Frequency"], + dummyregressor__sample_weight=df_train["Exposure"]) + + +def score_estimator(estimator, df_test): + """Score an estimator on the test set.""" + + y_pred = estimator.predict(df_test) + + print("MSE: %.3f" % + mean_squared_error(df_test["Frequency"], y_pred, + df_test["Exposure"])) + print("MAE: %.3f" % + mean_absolute_error(df_test["Frequency"], y_pred, + df_test["Exposure"])) + + # ignore non-positive predictions, as they are invalid for + # the Poisson deviance + mask = y_pred > 0 + if (~mask).any(): + warnings.warn("Estimator yields non-positive predictions for {} " + "samples out of {}. These will be ignored while " + "computing the Poisson deviance" + .format((~mask).sum(), mask.shape[0])) + + print("mean Poisson deviance: %.3f" % + mean_poisson_deviance(df_test["Frequency"][mask], + y_pred[mask], + df_test["Exposure"][mask])) + + +print("Constant mean frequency evaluation:") +score_estimator(dummy, df_test) + +############################################################################## +# We start by modeling the target variable with the least squares linear +# regression model, + +ridge = make_pipeline(linear_model_preprocessor, Ridge(alpha=1.0)) +ridge.fit(df_train, df_train["Frequency"], + ridge__sample_weight=df_train["Exposure"]) + +############################################################################## +# The Poisson deviance cannot be computed on non-positive values predicted by +# the model. For models that do return a few non-positive predictions +# (e.g. :class:`linear_model.Ridge`) we ignore the corresponding samples, +# meaning that the obtained Poisson deviance is approximate. An alternative +# approach could be to use :class:`compose.TransformedTargetRegressor` +# meta-estimator to map ``y_pred`` to a strictly positive domain. + +print("Ridge evaluation:") +score_estimator(ridge, df_test) + +############################################################################## +# Next we fit the Poisson regressor on the target variable. We set the +# regularization strength ``alpha`` to 1 over number of samples in oder to +# mimic the Ridge regressor whose L2 penalty term scales differently with the +# number of samples. + +poisson = make_pipeline( + linear_model_preprocessor, + PoissonRegressor(alpha=1/df_train.shape[0], max_iter=1000) +) +poisson.fit(df_train, df_train["Frequency"], + poissonregressor__sample_weight=df_train["Exposure"]) + +print("PoissonRegressor evaluation:") +score_estimator(poisson, df_test) + +############################################################################## +# Finally, we will consider a non-linear model, namely a random forest. Random +# forests do not require the categorical data to be one-hot encoded: instead, +# we can encode each category label with an arbitrary integer using +# :class:`preprocessing.OrdinalEncoder`. With this encoding, the forest will +# treat the categorical features as ordered features, which might not be always +# a desired behavior. However this effect is limited for deep enough trees +# which are able to recover the categorical nature of the features. The main +# advantage of the :class:`preprocessing.OrdinalEncoder` over the +# :class:`preprocessing.OneHotEncoder` is that it will make training faster. + +rf_preprocessor = ColumnTransformer( + [ + ("categorical", OrdinalEncoder(), + ["VehBrand", "VehPower", "VehGas", "Region", "Area"]), + ("numeric", "passthrough", + ["VehAge", "DrivAge", "BonusMalus", "Density"]), + ], + remainder="drop", +) +rf = make_pipeline( + rf_preprocessor, + RandomForestRegressor(min_weight_fraction_leaf=0.01, n_jobs=2) +) +rf.fit(df_train, df_train["Frequency"].values, + randomforestregressor__sample_weight=df_train["Exposure"].values) + + +print("RandomForestRegressor evaluation:") +score_estimator(rf, df_test) + + +############################################################################## +# Like the Ridge regression above, the random forest model minimizes the +# conditional squared error, too. However, because of a higher predictive +# power, it also results in a smaller Poisson deviance than the Poisson +# regression model. +# +# Evaluating models with a single train / test split is prone to random +# fluctuations. If computing resources allow, it should be verified that +# cross-validated performance metrics would lead to similar conclusions. +# +# The qualitative difference between these models can also be visualized by +# comparing the histogram of observed target values with that of predicted +# values: + +fig, axes = plt.subplots(2, 4, figsize=(16, 6), sharey=True) +fig.subplots_adjust(bottom=0.2) +n_bins = 20 +for row_idx, label, df in zip(range(2), + ["train", "test"], + [df_train, df_test]): + df["Frequency"].hist(bins=np.linspace(-1, 30, n_bins), + ax=axes[row_idx, 0]) + + axes[row_idx, 0].set_title("Data") + axes[row_idx, 0].set_yscale('log') + axes[row_idx, 0].set_xlabel("y (observed Frequency)") + axes[row_idx, 0].set_ylim([1e1, 5e5]) + axes[row_idx, 0].set_ylabel(label + " samples") + + for idx, model in enumerate([ridge, poisson, rf]): + y_pred = model.predict(df) + + pd.Series(y_pred).hist(bins=np.linspace(-1, 4, n_bins), + ax=axes[row_idx, idx+1]) + axes[row_idx, idx + 1].set( + title=model[-1].__class__.__name__, + yscale='log', + xlabel="y_pred (predicted expected Frequency)" + ) +plt.tight_layout() + +############################################################################## +# The experimental data presents a long tail distribution for ``y``. In all +# models we predict a mean expected value, so we will have necessarily fewer +# extreme values. Additionally, the normal distribution used in ``Ridge`` and +# ``RandomForestRegressor`` has a constant variance, while for the Poisson +# distribution used in ``PoissonRegressor``, the variance is proportional to +# the mean predicted value. +# +# Thus, among the considered estimators, ``PoissonRegressor`` is better suited +# for modeling the long tail distribution of the data as compared to the +# ``Ridge`` and ``RandomForestRegressor`` estimators. +# +# To ensure that estimators yield reasonable predictions for different +# policyholder types, we can bin test samples according to ``y_pred`` returned +# by each model. Then for each bin, we compare the mean predicted ``y_pred``, +# with the mean observed target: + + +def _mean_frequency_by_risk_group(y_true, y_pred, sample_weight=None, + n_bins=100): + """Compare predictions and observations for bins ordered by y_pred. + + We order the samples by ``y_pred`` and split it in bins. + In each bin the observed mean is compared with the predicted mean. + + Parameters + ---------- + y_true: array-like of shape (n_samples,) + Ground truth (correct) target values. + y_pred: array-like of shape (n_samples,) + Estimated target values. + sample_weight : array-like of shape (n_samples,) + Sample weights. + n_bins: int + Number of bins to use. + + Returns + ------- + bin_centers: ndarray of shape (n_bins,) + bin centers + y_true_bin: ndarray of shape (n_bins,) + average y_pred for each bin + y_pred_bin: ndarray of shape (n_bins,) + average y_pred for each bin + """ + idx_sort = np.argsort(y_pred) + bin_centers = np.arange(0, 1, 1/n_bins) + 0.5/n_bins + y_pred_bin = np.zeros(n_bins) + y_true_bin = np.zeros(n_bins) + + for n, sl in enumerate(gen_even_slices(len(y_true), n_bins)): + weights = sample_weight[idx_sort][sl] + y_pred_bin[n] = np.average( + y_pred[idx_sort][sl], weights=weights + ) + y_true_bin[n] = np.average( + y_true[idx_sort][sl], + weights=weights + ) + return bin_centers, y_true_bin, y_pred_bin + + +fig, ax = plt.subplots(nrows=1, ncols=3, figsize=(12, 3.5)) +plt.subplots_adjust(wspace=0.3) + +for axi, model in zip(ax, [ridge, poisson, rf]): + y_pred = model.predict(df_test) + + q, y_true_seg, y_pred_seg = _mean_frequency_by_risk_group( + df_test["Frequency"].values, + y_pred, + sample_weight=df_test["Exposure"].values, + n_bins=10) + + axi.plot(q, y_pred_seg, marker='o', linestyle="-", label="predictions") + axi.plot(q, y_true_seg, marker='x', linestyle="--", label="observations") + axi.set_xlim(0, 1.0) + axi.set_ylim(0, 0.6) + axi.set( + title=model[-1].__class__.__name__, + xlabel='Fraction of samples sorted by y_pred', + ylabel='Mean Frequency (y_pred)' + ) + axi.legend() +plt.tight_layout() + +############################################################################## +# The ``Ridge`` regression model can predict very low expected frequencies +# that do not match the data. It can therefore severly under-estimate the risk +# for some policyholders. +# +# ``PoissonRegressor`` and ``RandomForestRegressor`` show better consistency +# between predicted and observed targets, especially for low predicted target +# values. +# +# However, for some business applications, we are not necessarily interested +# in the ability of the model to predict the expected frequency value, but +# instead to predict which policyholder groups are the riskiest and which are +# the safest. In this case, the model evaluation would cast the problem as a +# ranking problem rather than a regression problem. +# +# To compare the 3 models within this perspective, one can plot the fraction of +# the number of claims vs the fraction of exposure for test samples ordered by +# the model predictions, from safest to riskiest according to each model: + + +def _cumulated_claims(y_true, y_pred, exposure): + idx_sort = np.argsort(y_pred) # from safest to riskiest + sorted_exposure = exposure[idx_sort] + sorted_frequencies = y_true[idx_sort] + cumulated_exposure = np.cumsum(sorted_exposure) + cumulated_exposure /= cumulated_exposure[-1] + cumulated_claims = np.cumsum(sorted_exposure * sorted_frequencies) + cumulated_claims /= cumulated_claims[-1] + return cumulated_exposure, cumulated_claims + + +fig, ax = plt.subplots(figsize=(8, 8)) + +for model in [ridge, poisson, rf]: + y_pred = model.predict(df_test) + cum_exposure, cum_claims = _cumulated_claims( + df_test["Frequency"].values, + y_pred, + df_test["Exposure"].values) + area = auc(cum_exposure, cum_claims) + label = "{} (area under curve: {:.3f})".format( + model[-1].__class__.__name__, area) + ax.plot(cum_exposure, cum_claims, linestyle="-", label=label) + +# Oracle model: y_pred == y_test +cum_exposure, cum_claims = _cumulated_claims( + df_test["Frequency"].values, + df_test["Frequency"].values, + df_test["Exposure"].values) +area = auc(cum_exposure, cum_claims) +label = "Oracle (area under curve: {:.3f})".format(area) +ax.plot(cum_exposure, cum_claims, linestyle="-.", color="gray", label=label) + +# Random Baseline +ax.plot([0, 1], [0, 1], linestyle="--", color="black", + label="Random baseline") +ax.set( + title="Cumulated number of claims by model", + xlabel='Fraction of exposure (from safest to riskiest)', + ylabel='Fraction of number of claims' +) +ax.legend(loc="upper left") + +############################################################################## +# This plot reveals that the random forest model is slightly better at ranking +# policyholders by risk profiles even if the absolute value of the predicted +# expected frequencies are less well calibrated than for the linear Poisson +# model. +# +# All three models are significantly better than chance but also very far from +# making perfect predictions. +# +# This last point is expected due to the nature of the problem: the occurrence +# of accidents is mostly dominated by circumstantial causes that are not +# captured in the columns of the dataset or that are indeed random. + +plt.show() diff --git a/examples/linear_model/plot_sparse_logistic_regression_20newsgroups.py b/examples/linear_model/plot_sparse_logistic_regression_20newsgroups.py index 78fdc64684550..7bfad99d991c5 100644 --- a/examples/linear_model/plot_sparse_logistic_regression_20newsgroups.py +++ b/examples/linear_model/plot_sparse_logistic_regression_20newsgroups.py @@ -1,7 +1,7 @@ """ -===================================================== -Multiclass sparse logisitic regression on newgroups20 -===================================================== +==================================================== +Multiclass sparse logistic regression on 20newgroups +==================================================== Comparison of multinomial logistic L1 vs one-versus-rest L1 logistic regression to classify documents from the newgroups20 dataset. Multinomial logistic @@ -42,7 +42,6 @@ # Turn down for faster run time n_samples = 10000 -# Memorized fetch_rcv1 for faster access X, y = fetch_20newsgroups_vectorized('all', return_X_y=True) X = X[:n_samples] y = y[:n_samples] diff --git a/examples/linear_model/plot_sparse_logistic_regression_mnist.py b/examples/linear_model/plot_sparse_logistic_regression_mnist.py index 56b5457c6a27e..ab3749fb5e7f8 100644 --- a/examples/linear_model/plot_sparse_logistic_regression_mnist.py +++ b/examples/linear_model/plot_sparse_logistic_regression_mnist.py @@ -1,6 +1,6 @@ """ ===================================================== -MNIST classfification using multinomial logistic + L1 +MNIST classification using multinomial logistic + L1 ===================================================== Here we fit a multinomial logistic regression with L1 penalty on a subset of diff --git a/examples/linear_model/plot_tweedie_regression_insurance_claims.py b/examples/linear_model/plot_tweedie_regression_insurance_claims.py new file mode 100644 index 0000000000000..ccd18c8efff99 --- /dev/null +++ b/examples/linear_model/plot_tweedie_regression_insurance_claims.py @@ -0,0 +1,596 @@ +""" +====================================== +Tweedie regression on insurance claims +====================================== + +This example illustrates the use of Poisson, Gamma and Tweedie regression on +the `French Motor Third-Party Liability Claims dataset +`_, and is inspired by an R tutorial [1]_. + +In this dataset, each sample corresponds to an insurance policy, i.e. a +contract within an insurance company and an individual (policyholder). +Available features include driver age, vehicle age, vehicle power, etc. + +A few definitions: a *claim* is the request made by a policyholder to the +insurer to compensate for a loss covered by the insurance. The *claim amount* +is the amount of money that the insurer must pay. The *exposure* is the +duration of the insurance coverage of a given policy, in years. + +Here our goal goal is to predict the expected +value, i.e. the mean, of the total claim amount per exposure unit also +referred to as the pure premium. + +There are several possibilities to do that, two of which are: + +1. Model the number of claims with a Poisson distribution, and the average + claim amount per claim, also known as severity, as a Gamma distribution + and multiply the predictions of both in order to get the total claim + amount. +2. Model the total claim amount per exposure directly, typically with a Tweedie + distribution of Tweedie power :math:`p \\in (1, 2)`. + +In this example we will illustrate both approaches. We start by defining a few +helper functions for loading the data and visualizing results. + +.. [1] A. Noll, R. Salzmann and M.V. Wuthrich, Case Study: French Motor + Third-Party Liability Claims (November 8, 2018). `doi:10.2139/ssrn.3164764 + `_ + +""" +print(__doc__) + +# Authors: Christian Lorentzen +# Roman Yurchak +# Olivier Grisel +# License: BSD 3 clause +from functools import partial + +import numpy as np +import matplotlib.pyplot as plt +import pandas as pd + +from sklearn.datasets import fetch_openml +from sklearn.compose import ColumnTransformer +from sklearn.linear_model import PoissonRegressor, GammaRegressor +from sklearn.linear_model import TweedieRegressor +from sklearn.metrics import mean_tweedie_deviance +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import FunctionTransformer, OneHotEncoder +from sklearn.preprocessing import StandardScaler, KBinsDiscretizer + +from sklearn.metrics import mean_absolute_error, mean_squared_error, auc + + +def load_mtpl2(n_samples=100000): + """Fetch the French Motor Third-Party Liability Claims dataset. + + Parameters + ---------- + n_samples: int, default=100000 + number of samples to select (for faster run time). Full dataset has + 678013 samples. + """ + # freMTPL2freq dataset from https://www.openml.org/d/41214 + df_freq = fetch_openml(data_id=41214, as_frame=True)['data'] + df_freq['IDpol'] = df_freq['IDpol'].astype(np.int) + df_freq.set_index('IDpol', inplace=True) + + # freMTPL2sev dataset from https://www.openml.org/d/41215 + df_sev = fetch_openml(data_id=41215, as_frame=True)['data'] + + # sum ClaimAmount over identical IDs + df_sev = df_sev.groupby('IDpol').sum() + + df = df_freq.join(df_sev, how="left") + df["ClaimAmount"].fillna(0, inplace=True) + + # unquote string fields + for column_name in df.columns[df.dtypes.values == np.object]: + df[column_name] = df[column_name].str.strip("'") + return df.iloc[:n_samples] + + +def plot_obs_pred(df, feature, weight, observed, predicted, y_label=None, + title=None, ax=None, fill_legend=False): + """Plot observed and predicted - aggregated per feature level. + + Parameters + ---------- + df : DataFrame + input data + feature: str + a column name of df for the feature to be plotted + weight : str + column name of df with the values of weights or exposure + observed : str + a column name of df with the observed target + predicted : DataFrame + a dataframe, with the same index as df, with the predicted target + fill_legend : bool, default=False + whether to show fill_between legend + """ + # aggregate observed and predicted variables by feature level + df_ = df.loc[:, [feature, weight]].copy() + df_["observed"] = df[observed] * df[weight] + df_["predicted"] = predicted * df[weight] + df_ = ( + df_.groupby([feature])[weight, "observed", "predicted"] + .sum() + .assign(observed=lambda x: x["observed"] / x[weight]) + .assign(predicted=lambda x: x["predicted"] / x[weight]) + ) + + ax = df_.loc[:, ["observed", "predicted"]].plot(style=".", ax=ax) + y_max = df_.loc[:, ["observed", "predicted"]].values.max() * 0.8 + p2 = ax.fill_between( + df_.index, + 0, + y_max * df_[weight] / df_[weight].values.max(), + color="g", + alpha=0.1, + ) + if fill_legend: + ax.legend([p2], ["{} distribution".format(feature)]) + ax.set( + ylabel=y_label if y_label is not None else None, + title=title if title is not None else "Train: Observed vs Predicted", + ) + + +def score_estimator( + estimator, X_train, X_test, df_train, df_test, target, weights, + tweedie_powers=None, +): + """Evaluate an estimator on train and test sets with different metrics""" + + metrics = [ + ("D² explained", None), # Use default scorer if it exists + ("mean abs. error", mean_absolute_error), + ("mean squared error", mean_squared_error), + ] + if tweedie_powers: + metrics += [( + "mean Tweedie dev p={:.4f}".format(power), + partial(mean_tweedie_deviance, power=power) + ) for power in tweedie_powers] + + res = [] + for subset_label, X, df in [ + ("train", X_train, df_train), + ("test", X_test, df_test), + ]: + y, _weights = df[target], df[weights] + for score_label, metric in metrics: + if isinstance(estimator, tuple) and len(estimator) == 2: + # Score the model consisting of the product of frequency and + # severity models. + est_freq, est_sev = estimator + y_pred = est_freq.predict(X) * est_sev.predict(X) + else: + y_pred = estimator.predict(X) + + if metric is None: + if not hasattr(estimator, "score"): + continue + score = estimator.score(X, y, _weights) + else: + score = metric(y, y_pred, _weights) + + res.append( + {"subset": subset_label, "metric": score_label, "score": score} + ) + + res = ( + pd.DataFrame(res) + .set_index(["metric", "subset"]) + .score.unstack(-1) + .round(4) + .loc[:, ['train', 'test']] + ) + return res + + +############################################################################## +# Loading datasets, basic feature extraction and target definitions +# ----------------------------------------------------------------- +# +# We construct the freMTPL2 dataset by joining the freMTPL2freq table, +# containing the number of claims (``ClaimNb``), with the freMTPL2sev table, +# containing the claim amount (``ClaimAmount``) for the same policy ids +# (``IDpol``). + +df = load_mtpl2(n_samples=60000) + +# Note: filter out claims with zero amount, as the severity model +# requires strictly positive target values. +df.loc[(df["ClaimAmount"] == 0) & (df["ClaimNb"] >= 1), "ClaimNb"] = 0 + +# Correct for unreasonable observations (that might be data error) +# and a few exceptionally large claim amounts +df["ClaimNb"] = df["ClaimNb"].clip(upper=4) +df["Exposure"] = df["Exposure"].clip(upper=1) +df["ClaimAmount"] = df["ClaimAmount"].clip(upper=200000) + +log_scale_transformer = make_pipeline( + FunctionTransformer(func=np.log), + StandardScaler() +) + +column_trans = ColumnTransformer( + [ + ("binned_numeric", KBinsDiscretizer(n_bins=10), + ["VehAge", "DrivAge"]), + ("onehot_categorical", OneHotEncoder(), + ["VehBrand", "VehPower", "VehGas", "Region", "Area"]), + ("passthrough_numeric", "passthrough", + ["BonusMalus"]), + ("log_scaled_numeric", log_scale_transformer, + ["Density"]), + ], + remainder="drop", +) +X = column_trans.fit_transform(df) + +# Insurances companies are interested in modeling the Pure Premium, that is +# the expected total claim amount per unit of exposure for each policyholder +# in their portfolio: +df["PurePremium"] = df["ClaimAmount"] / df["Exposure"] + +# This can be indirectly approximated by a 2-step modeling: the product of the +# Frequency times the average claim amount per claim: +df["Frequency"] = df["ClaimNb"] / df["Exposure"] +df["AvgClaimAmount"] = df["ClaimAmount"] / np.fmax(df["ClaimNb"], 1) + +with pd.option_context("display.max_columns", 15): + print(df[df.ClaimAmount > 0].head()) + +############################################################################## +# +# Frequency model -- Poisson distribution +# --------------------------------------- +# +# The number of claims (``ClaimNb``) is a positive integer (0 included). +# Thus, this target can be modelled by a Poisson distribution. +# It is then assumed to be the number of discrete events occuring with a +# constant rate in a given time interval (``Exposure``, in units of years). +# Here we model the frequency ``y = ClaimNb / Exposure``, which is still a +# (scaled) Poisson distribution, and use ``Exposure`` as `sample_weight`. + +df_train, df_test, X_train, X_test = train_test_split(df, X, random_state=0) + +# The parameters of the model are estimated by minimizing the Poisson deviance +# on the training set via a quasi-Newton solver: l-BFGS. Some of the features +# are collinear, we use a weak penalization to avoid numerical issues. +glm_freq = PoissonRegressor(alpha=1e-3, max_iter=400) +glm_freq.fit(X_train, df_train["Frequency"], + sample_weight=df_train["Exposure"]) + +scores = score_estimator( + glm_freq, + X_train, + X_test, + df_train, + df_test, + target="Frequency", + weights="Exposure", +) +print("Evaluation of PoissonRegressor on target Frequency") +print(scores) + +############################################################################## +# We can visually compare observed and predicted values, aggregated by the +# drivers age (``DrivAge``), vehicle age (``VehAge``) and the insurance +# bonus/malus (``BonusMalus``). + +fig, ax = plt.subplots(ncols=2, nrows=2, figsize=(16, 8)) +fig.subplots_adjust(hspace=0.3, wspace=0.2) + +plot_obs_pred( + df=df_train, + feature="DrivAge", + weight="Exposure", + observed="Frequency", + predicted=glm_freq.predict(X_train), + y_label="Claim Frequency", + title="train data", + ax=ax[0, 0], +) + +plot_obs_pred( + df=df_test, + feature="DrivAge", + weight="Exposure", + observed="Frequency", + predicted=glm_freq.predict(X_test), + y_label="Claim Frequency", + title="test data", + ax=ax[0, 1], + fill_legend=True +) + +plot_obs_pred( + df=df_test, + feature="VehAge", + weight="Exposure", + observed="Frequency", + predicted=glm_freq.predict(X_test), + y_label="Claim Frequency", + title="test data", + ax=ax[1, 0], + fill_legend=True +) + +plot_obs_pred( + df=df_test, + feature="BonusMalus", + weight="Exposure", + observed="Frequency", + predicted=glm_freq.predict(X_test), + y_label="Claim Frequency", + title="test data", + ax=ax[1, 1], + fill_legend=True +) + + +############################################################################## +# According to the observed data, the frequency of accidents is higher for +# drivers younger than 30 years old, and is positively correlated with the +# `BonusMalus` variable. Our model is able to mostly correctly model this +# behaviour. +# +# Severity Model - Gamma distribution +# ------------------------------------ +# The mean claim amount or severity (`AvgClaimAmount`) can be empirically +# shown to follow approximately a Gamma distribution. We fit a GLM model for +# the severity with the same features as the frequency model. +# +# Note: +# +# - We filter out ``ClaimAmount == 0`` as the Gamma distribution has support +# on :math:`(0, \infty)`, not :math:`[0, \infty)`. +# - We use ``ClaimNb`` as `sample_weight` to account for policies that contain +# more than one claim. + +mask_train = df_train["ClaimAmount"] > 0 +mask_test = df_test["ClaimAmount"] > 0 + +glm_sev = GammaRegressor(alpha=10., max_iter=10000) + +glm_sev.fit( + X_train[mask_train.values], + df_train.loc[mask_train, "AvgClaimAmount"], + sample_weight=df_train.loc[mask_train, "ClaimNb"], +) + +scores = score_estimator( + glm_sev, + X_train[mask_train.values], + X_test[mask_test.values], + df_train[mask_train], + df_test[mask_test], + target="AvgClaimAmount", + weights="ClaimNb", +) +print("Evaluation of GammaRegressor on target AvgClaimAmount") +print(scores) + +############################################################################## +# Here, the scores for the test data call for caution as they are +# significantly worse than for the training data indicating an overfit despite +# the strong regularization. +# +# Note that the resulting model is the average claim amount per claim. As +# such, it is conditional on having at least one claim, and cannot be used to +# predict the average claim amount per policy in general. + +print("Mean AvgClaim Amount per policy: %.2f " + % df_train["AvgClaimAmount"].mean()) +print("Mean AvgClaim Amount | NbClaim > 0: %.2f" + % df_train["AvgClaimAmount"][df_train["AvgClaimAmount"] > 0].mean()) +print("Predicted Mean AvgClaim Amount | NbClaim > 0: %.2f" + % glm_sev.predict(X_train).mean()) + + +############################################################################## +# We can visually compare observed and predicted values, aggregated for +# the drivers age (``DrivAge``). + +fig, ax = plt.subplots(ncols=1, nrows=2, figsize=(16, 6)) + +plot_obs_pred( + df=df_train.loc[mask_train], + feature="DrivAge", + weight="Exposure", + observed="AvgClaimAmount", + predicted=glm_sev.predict(X_train[mask_train.values]), + y_label="Average Claim Severity", + title="train data", + ax=ax[0], +) + +plot_obs_pred( + df=df_test.loc[mask_test], + feature="DrivAge", + weight="Exposure", + observed="AvgClaimAmount", + predicted=glm_sev.predict(X_test[mask_test.values]), + y_label="Average Claim Severity", + title="test data", + ax=ax[1], + fill_legend=True +) +plt.tight_layout() + +############################################################################## +# Overall, the drivers age (``DrivAge``) has a weak impact on the claim +# severity, both in observed and predicted data. +# +# Pure Premium Modeling via a Product Model vs single TweedieRegressor +# -------------------------------------------------------------------- +# As mentioned in the introduction, the total claim amount per unit of +# exposure can be modeled as the product of the prediction of the +# frequency model by the prediction of the severity model. +# +# Alternatively, one can directly model the total loss with a unique +# Compound Poisson Gamma generalized linear model (with a log link function). +# This model is a special case of the Tweedie GLM with a "power" parameter +# :math:`p \in (1, 2)`. Here, we fix apriori the `power` parameter of the +# Tweedie model to some arbitrary value (1.9) in the valid range. Ideally one +# would select this value via grid-search by minimizing the negative +# log-likelihood of the Tweedie model, but unfortunately the current +# implementation does not allow for this (yet). +# +# We will compare the performance of both approaches. +# To quantify the performance of both models, one can compute +# the mean deviance of the train and test data assuming a Compound +# Poisson-Gamma distribution of the total claim amount. This is equivalent to +# a Tweedie distribution with a `power` parameter between 1 and 2. +# +# The :func:`sklearn.metrics.mean_tweedie_deviance` depends on a `power` +# parameter. As we do not know the true value of the `power` parameter, we here +# compute the mean deviances for a grid of possible values, and compare the +# models side by side, i.e. we compare them at identical values of `power`. +# Ideally, we hope that one model will be consistently better than the other, +# regardless of `power`. + +glm_pure_premium = TweedieRegressor(power=1.9, alpha=.1, max_iter=10000) +glm_pure_premium.fit(X_train, df_train["PurePremium"], + sample_weight=df_train["Exposure"]) + +tweedie_powers = [1.5, 1.7, 1.8, 1.9, 1.99, 1.999, 1.9999] + +scores_product_model = score_estimator( + (glm_freq, glm_sev), + X_train, + X_test, + df_train, + df_test, + target="PurePremium", + weights="Exposure", + tweedie_powers=tweedie_powers, +) + +scores_glm_pure_premium = score_estimator( + glm_pure_premium, + X_train, + X_test, + df_train, + df_test, + target="PurePremium", + weights="Exposure", + tweedie_powers=tweedie_powers +) + +scores = pd.concat([scores_product_model, scores_glm_pure_premium], + axis=1, sort=True, + keys=('Product Model', 'TweedieRegressor')) +print("Evaluation of the Product Model and the Tweedie Regressor " + "on target PurePremium") +with pd.option_context('display.expand_frame_repr', False): + print(scores) + +############################################################################## +# In this example, both modeling approaches yield comparable performance +# metrics. For implementation reasons, the percentage of explained variance +# :math:`D^2` is not available for the product model. +# +# We can additionally validate these models by comparing observed and +# predicted total claim amount over the test and train subsets. We see that, +# on average, both model tend to underestimate the total claim (but this +# behavior depends on the amount of regularization). + +res = [] +for subset_label, X, df in [ + ("train", X_train, df_train), + ("test", X_test, df_test), +]: + exposure = df["Exposure"].values + res.append( + { + "subset": subset_label, + "observed": df["ClaimAmount"].values.sum(), + "predicted, frequency*severity model": np.sum( + exposure * glm_freq.predict(X) * glm_sev.predict(X) + ), + "predicted, tweedie, power=%.2f" + % glm_pure_premium.power: np.sum( + exposure * glm_pure_premium.predict(X)), + } + ) + +print(pd.DataFrame(res).set_index("subset").T) + +############################################################################## +# Finally, we can compare the two models using a plot of cumulated claims: for +# each model, the policyholders are ranked from safest to riskiest and the +# fraction of observed total cumulated claims is plotted on the y axis. This +# plot is often called the ordered Lorenz curve of the model. +# +# The Gini coefficient (based on the area under the curve) can be used as a +# model selection metric to quantify the ability of the model to rank +# policyholders. Note that this metric does not reflect the ability of the +# models to make accurate predictions in terms of absolute value of total +# claim amounts but only in terms of relative amounts as a ranking metric. +# +# Both models are able to rank policyholders by risky-ness significantly +# better than chance although they are also both far from perfect due to the +# natural difficulty of the prediction problem from few features. +# +# Note that the Gini index only characterize the ranking performance of the +# model but not its calibration: any monotonic transformation of the +# predictions leaves the Gini index of the model unchanged. +# +# Finally one should highlight that the Compound Poisson Gamma model that +# is directly fit on the pure premium is operationally simpler to develop and +# maintain as it consists in a single scikit-learn estimator instead of a +# pair of models, each with its own set of hyperparameters. + + +def lorenz_curve(y_true, y_pred, exposure): + y_true, y_pred = np.asarray(y_true), np.asarray(y_pred) + exposure = np.asarray(exposure) + + # order samples by increasing predicted risk: + ranking = np.argsort(y_pred) + ranked_exposure = exposure[ranking] + ranked_pure_premium = y_true[ranking] + cumulated_claim_amount = np.cumsum(ranked_pure_premium * ranked_exposure) + cumulated_claim_amount /= cumulated_claim_amount[-1] + cumulated_samples = np.linspace(0, 1, len(cumulated_claim_amount)) + return cumulated_samples, cumulated_claim_amount + + +fig, ax = plt.subplots(figsize=(8, 8)) + +y_pred_product = glm_freq.predict(X_test) * glm_sev.predict(X_test) +y_pred_total = glm_pure_premium.predict(X_test) + +for label, y_pred in [("Frequency * Severity model", y_pred_product), + ("Compound Poisson Gamma", y_pred_total)]: + ordered_samples, cum_claims = lorenz_curve( + df_test["PurePremium"], y_pred, df_test["Exposure"]) + gini = 1 - 2 * auc(ordered_samples, cum_claims) + label += " (Gini index: {:.3f})".format(gini) + ax.plot(ordered_samples, cum_claims, linestyle="-", label=label) + +# Oracle model: y_pred == y_test +ordered_samples, cum_claims = lorenz_curve( + df_test["PurePremium"], df_test["PurePremium"], df_test["Exposure"]) +gini = 1 - 2 * auc(ordered_samples, cum_claims) +label = "Oracle (Gini index: {:.3f})".format(gini) +ax.plot(ordered_samples, cum_claims, linestyle="-.", color="gray", + label=label) + +# Random baseline +ax.plot([0, 1], [0, 1], linestyle="--", color="black", + label="Random baseline") +ax.set( + title="Lorenz Curves", + xlabel=('Fraction of policyholders\n' + '(ordered by model from safest to riskiest)'), + ylabel='Fraction of total claim amount' +) +ax.legend(loc="upper left") +plt.plot() diff --git a/examples/manifold/plot_compare_methods.py b/examples/manifold/plot_compare_methods.py index 7c5f3b6200635..ed01e8ac19b89 100644 --- a/examples/manifold/plot_compare_methods.py +++ b/examples/manifold/plot_compare_methods.py @@ -23,6 +23,8 @@ print(__doc__) +from collections import OrderedDict +from functools import partial from time import time import matplotlib.pyplot as plt @@ -39,81 +41,43 @@ n_neighbors = 10 n_components = 2 +# Create figure fig = plt.figure(figsize=(15, 8)) -plt.suptitle("Manifold Learning with %i points, %i neighbors" +fig.suptitle("Manifold Learning with %i points, %i neighbors" % (1000, n_neighbors), fontsize=14) - +# Add 3d scatter plot ax = fig.add_subplot(251, projection='3d') ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=color, cmap=plt.cm.Spectral) ax.view_init(4, -72) -methods = ['standard', 'ltsa', 'hessian', 'modified'] -labels = ['LLE', 'LTSA', 'Hessian LLE', 'Modified LLE'] - -for i, method in enumerate(methods): +# Set-up manifold methods +LLE = partial(manifold.LocallyLinearEmbedding, + n_neighbors, n_components, eigen_solver='auto') + +methods = OrderedDict() +methods['LLE'] = LLE(method='standard') +methods['LTSA'] = LLE(method='ltsa') +methods['Hessian LLE'] = LLE(method='hessian') +methods['Modified LLE'] = LLE(method='modified') +methods['Isomap'] = manifold.Isomap(n_neighbors, n_components) +methods['MDS'] = manifold.MDS(n_components, max_iter=100, n_init=1) +methods['SE'] = manifold.SpectralEmbedding(n_components=n_components, + n_neighbors=n_neighbors) +methods['t-SNE'] = manifold.TSNE(n_components=n_components, init='pca', + random_state=0) + +# Plot results +for i, (label, method) in enumerate(methods.items()): t0 = time() - Y = manifold.LocallyLinearEmbedding(n_neighbors, n_components, - eigen_solver='auto', - method=method).fit_transform(X) + Y = method.fit_transform(X) t1 = time() - print("%s: %.2g sec" % (methods[i], t1 - t0)) - - ax = fig.add_subplot(252 + i) - plt.scatter(Y[:, 0], Y[:, 1], c=color, cmap=plt.cm.Spectral) - plt.title("%s (%.2g sec)" % (labels[i], t1 - t0)) + print("%s: %.2g sec" % (label, t1 - t0)) + ax = fig.add_subplot(2, 5, 2 + i + (i > 3)) + ax.scatter(Y[:, 0], Y[:, 1], c=color, cmap=plt.cm.Spectral) + ax.set_title("%s (%.2g sec)" % (label, t1 - t0)) ax.xaxis.set_major_formatter(NullFormatter()) ax.yaxis.set_major_formatter(NullFormatter()) - plt.axis('tight') - -t0 = time() -Y = manifold.Isomap(n_neighbors, n_components).fit_transform(X) -t1 = time() -print("Isomap: %.2g sec" % (t1 - t0)) -ax = fig.add_subplot(257) -plt.scatter(Y[:, 0], Y[:, 1], c=color, cmap=plt.cm.Spectral) -plt.title("Isomap (%.2g sec)" % (t1 - t0)) -ax.xaxis.set_major_formatter(NullFormatter()) -ax.yaxis.set_major_formatter(NullFormatter()) -plt.axis('tight') - - -t0 = time() -mds = manifold.MDS(n_components, max_iter=100, n_init=1) -Y = mds.fit_transform(X) -t1 = time() -print("MDS: %.2g sec" % (t1 - t0)) -ax = fig.add_subplot(258) -plt.scatter(Y[:, 0], Y[:, 1], c=color, cmap=plt.cm.Spectral) -plt.title("MDS (%.2g sec)" % (t1 - t0)) -ax.xaxis.set_major_formatter(NullFormatter()) -ax.yaxis.set_major_formatter(NullFormatter()) -plt.axis('tight') - - -t0 = time() -se = manifold.SpectralEmbedding(n_components=n_components, - n_neighbors=n_neighbors) -Y = se.fit_transform(X) -t1 = time() -print("SpectralEmbedding: %.2g sec" % (t1 - t0)) -ax = fig.add_subplot(259) -plt.scatter(Y[:, 0], Y[:, 1], c=color, cmap=plt.cm.Spectral) -plt.title("SpectralEmbedding (%.2g sec)" % (t1 - t0)) -ax.xaxis.set_major_formatter(NullFormatter()) -ax.yaxis.set_major_formatter(NullFormatter()) -plt.axis('tight') - -t0 = time() -tsne = manifold.TSNE(n_components=n_components, init='pca', random_state=0) -Y = tsne.fit_transform(X) -t1 = time() -print("t-SNE: %.2g sec" % (t1 - t0)) -ax = fig.add_subplot(2, 5, 10) -plt.scatter(Y[:, 0], Y[:, 1], c=color, cmap=plt.cm.Spectral) -plt.title("t-SNE (%.2g sec)" % (t1 - t0)) -ax.xaxis.set_major_formatter(NullFormatter()) -ax.yaxis.set_major_formatter(NullFormatter()) -plt.axis('tight') + ax.axis('tight') plt.show() diff --git a/examples/manifold/plot_t_sne_perplexity.py b/examples/manifold/plot_t_sne_perplexity.py index cda936cf72142..dd7b4d1f21a09 100644 --- a/examples/manifold/plot_t_sne_perplexity.py +++ b/examples/manifold/plot_t_sne_perplexity.py @@ -6,7 +6,7 @@ An illustration of t-SNE on the two concentric circles and the S-curve datasets for different perplexity values. -We observe a tendency towards clearer shapes as the preplexity value increases. +We observe a tendency towards clearer shapes as the perplexity value increases. The size, the distance and the shape of clusters may vary upon initialization, perplexity values and does not always convey a meaning. diff --git a/examples/mixture/plot_gmm_sin.py b/examples/mixture/plot_gmm_sin.py index f5fb2ded45120..1d436b93d15cc 100644 --- a/examples/mixture/plot_gmm_sin.py +++ b/examples/mixture/plot_gmm_sin.py @@ -26,7 +26,7 @@ similar to the first model where we arbitrarily decided to fix the number of components to 10. -Which model is the best is a matter of subjective judgement: do we want to +Which model is the best is a matter of subjective judgment: do we want to favor models that only capture the big picture to summarize and explain most of the structure of the data while ignoring the details or do we prefer models that closely follow the high density regions of the signal? diff --git a/examples/model_selection/plot_confusion_matrix.py b/examples/model_selection/plot_confusion_matrix.py index 8e4aa73149505..5bed1a2ccec38 100644 --- a/examples/model_selection/plot_confusion_matrix.py +++ b/examples/model_selection/plot_confusion_matrix.py @@ -31,8 +31,7 @@ from sklearn import svm, datasets from sklearn.model_selection import train_test_split -from sklearn.metrics import confusion_matrix -from sklearn.utils.multiclass import unique_labels +from sklearn.metrics import plot_confusion_matrix # import some data to play with iris = datasets.load_iris() @@ -45,72 +44,21 @@ # Run classifier, using a model that is too regularized (C too low) to see # the impact on the results -classifier = svm.SVC(kernel='linear', C=0.01) -y_pred = classifier.fit(X_train, y_train).predict(X_test) - - -def plot_confusion_matrix(y_true, y_pred, classes, - normalize=False, - title=None, - cmap=plt.cm.Blues): - """ - This function prints and plots the confusion matrix. - Normalization can be applied by setting `normalize=True`. - """ - if not title: - if normalize: - title = 'Normalized confusion matrix' - else: - title = 'Confusion matrix, without normalization' - - # Compute confusion matrix - cm = confusion_matrix(y_true, y_pred) - # Only use the labels that appear in the data - classes = classes[unique_labels(y_true, y_pred)] - if normalize: - cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] - print("Normalized confusion matrix") - else: - print('Confusion matrix, without normalization') - - print(cm) - - fig, ax = plt.subplots() - im = ax.imshow(cm, interpolation='nearest', cmap=cmap) - ax.figure.colorbar(im, ax=ax) - # We want to show all ticks... - ax.set(xticks=np.arange(cm.shape[1]), - yticks=np.arange(cm.shape[0]), - # ... and label them with the respective list entries - xticklabels=classes, yticklabels=classes, - title=title, - ylabel='True label', - xlabel='Predicted label') - - # Rotate the tick labels and set their alignment. - plt.setp(ax.get_xticklabels(), rotation=45, ha="right", - rotation_mode="anchor") - - # Loop over data dimensions and create text annotations. - fmt = '.2f' if normalize else 'd' - thresh = cm.max() / 2. - for i in range(cm.shape[0]): - for j in range(cm.shape[1]): - ax.text(j, i, format(cm[i, j], fmt), - ha="center", va="center", - color="white" if cm[i, j] > thresh else "black") - fig.tight_layout() - return ax - +classifier = svm.SVC(kernel='linear', C=0.01).fit(X_train, y_train) np.set_printoptions(precision=2) # Plot non-normalized confusion matrix -plot_confusion_matrix(y_test, y_pred, classes=class_names, - title='Confusion matrix, without normalization') - -# Plot normalized confusion matrix -plot_confusion_matrix(y_test, y_pred, classes=class_names, normalize=True, - title='Normalized confusion matrix') +titles_options = [("Confusion matrix, without normalization", None), + ("Normalized confusion matrix", 'true')] +for title, normalize in titles_options: + disp = plot_confusion_matrix(classifier, X_test, y_test, + display_labels=class_names, + cmap=plt.cm.Blues, + normalize=normalize) + disp.ax_.set_title(title) + + print(title) + print(disp.confusion_matrix) plt.show() diff --git a/examples/model_selection/plot_cv_predict.py b/examples/model_selection/plot_cv_predict.py index 2cff4c8c05b39..ee3e82f42cba1 100644 --- a/examples/model_selection/plot_cv_predict.py +++ b/examples/model_selection/plot_cv_predict.py @@ -14,7 +14,7 @@ import matplotlib.pyplot as plt lr = linear_model.LinearRegression() -X, y = datasets.load_boston(return_X_y=True) +X, y = datasets.load_diabetes(return_X_y=True) # cross_val_predict returns an array of the same size as `y` where each entry # is a prediction obtained by cross validation: diff --git a/examples/model_selection/plot_roc.py b/examples/model_selection/plot_roc.py index d995c5c653ce4..d32ab06f7bf25 100644 --- a/examples/model_selection/plot_roc.py +++ b/examples/model_selection/plot_roc.py @@ -151,7 +151,7 @@ # ......................................... # The :func:`sklearn.metrics.roc_auc_score` function can be used for # multi-class classification. The multi-class One-vs-One scheme compares every -# unique pairwise combination of classes. In this section, we calcuate the AUC +# unique pairwise combination of classes. In this section, we calculate the AUC # using the OvR and OvO schemes. We report a macro average, and a # prevalence-weighted average. y_prob = classifier.predict_proba(X_test) diff --git a/examples/neighbors/approximate_nearest_neighbors.py b/examples/neighbors/approximate_nearest_neighbors.py index 16b43e347068c..bd71472f447fb 100644 --- a/examples/neighbors/approximate_nearest_neighbors.py +++ b/examples/neighbors/approximate_nearest_neighbors.py @@ -209,9 +209,9 @@ def test_transformers(): def load_mnist(n_samples): """Load MNIST, shuffle the data, and return only n_samples.""" - mnist = fetch_openml(data_id=41063) - X, y = shuffle(mnist.data, mnist.target, random_state=42) - return X[:n_samples], y[:n_samples] + mnist = fetch_openml("mnist_784") + X, y = shuffle(mnist.data, mnist.target, random_state=2) + return X[:n_samples] / 255, y[:n_samples] def run_benchmark(): @@ -278,8 +278,8 @@ def run_benchmark(): # plot TSNE embedding which should be very similar across methods if 'TSNE' in transformer_name: axes[i_ax].set_title(transformer_name + '\non ' + dataset_name) - axes[i_ax].scatter(Xt[:, 0], Xt[:, 1], c=y, alpha=0.2, - cmap=plt.cm.viridis) + axes[i_ax].scatter(Xt[:, 0], Xt[:, 1], c=y.astype(np.int32), + alpha=0.2, cmap=plt.cm.viridis) axes[i_ax].xaxis.set_major_formatter(NullFormatter()) axes[i_ax].yaxis.set_major_formatter(NullFormatter()) axes[i_ax].axis('tight') diff --git a/examples/neural_networks/plot_mlp_alpha.py b/examples/neural_networks/plot_mlp_alpha.py index 7f718539131d4..4d18fdcaa314c 100644 --- a/examples/neural_networks/plot_mlp_alpha.py +++ b/examples/neural_networks/plot_mlp_alpha.py @@ -28,6 +28,7 @@ from sklearn.preprocessing import StandardScaler from sklearn.datasets import make_moons, make_circles, make_classification from sklearn.neural_network import MLPClassifier +from sklearn.pipeline import make_pipeline h = .02 # step size in the mesh @@ -36,8 +37,13 @@ classifiers = [] for i in alphas: - classifiers.append(MLPClassifier(solver='lbfgs', alpha=i, random_state=1, - hidden_layer_sizes=[100, 100])) + classifiers.append(make_pipeline( + StandardScaler(), + MLPClassifier(solver='lbfgs', alpha=i, + random_state=1, max_iter=2000, + early_stopping=True, + hidden_layer_sizes=[100, 100]) + )) X, y = make_classification(n_features=2, n_redundant=0, n_informative=2, random_state=0, n_clusters_per_class=1) diff --git a/examples/plot_changed_only_pprint_parameter.py b/examples/plot_changed_only_pprint_parameter.py index 1a687cff046d8..a35471105b6c1 100644 --- a/examples/plot_changed_only_pprint_parameter.py +++ b/examples/plot_changed_only_pprint_parameter.py @@ -5,7 +5,7 @@ This example illustrates the use of the print_changed_only global parameter. -Setting print_changed_only to True will alterate the representation of +Setting print_changed_only to True will alternate the representation of estimators to only show the parameters that have been set to non-default values. This can be used to have more compact representations. """ diff --git a/examples/plot_partial_dependence_visualization_api.py b/examples/plot_partial_dependence_visualization_api.py index 40d833d39b12f..8ccb225afc2d0 100644 --- a/examples/plot_partial_dependence_visualization_api.py +++ b/examples/plot_partial_dependence_visualization_api.py @@ -17,7 +17,7 @@ import pandas as pd import matplotlib.pyplot as plt -from sklearn.datasets import load_boston +from sklearn.datasets import load_diabetes from sklearn.neural_network import MLPRegressor from sklearn.preprocessing import StandardScaler from sklearn.pipeline import make_pipeline @@ -26,15 +26,15 @@ ############################################################################## -# Train models on the boston housing price dataset +# Train models on the diabetes dataset # ================================================ # -# First, we train a decision tree and a multi-layer perceptron on the boston -# housing price dataset. +# First, we train a decision tree and a multi-layer perceptron on the diabetes +# dataset. -boston = load_boston() -X = pd.DataFrame(boston.data, columns=boston.feature_names) -y = boston.target +diabetes = load_diabetes() +X = pd.DataFrame(diabetes.data, columns=diabetes.feature_names) +y = diabetes.target tree = DecisionTreeRegressor() mlp = make_pipeline(StandardScaler(), @@ -43,19 +43,18 @@ tree.fit(X, y) mlp.fit(X, y) - ############################################################################## # Plotting partial dependence for two features # ============================================ # -# We plot partial dependence curves for features "LSTAT" and "RM" for -# the decision tree. With two features, +# We plot partial dependence curves for features "age" and "bmi" (body mass +# index) for the decision tree. With two features, # :func:`~sklearn.inspection.plot_partial_dependence` expects to plot two # curves. Here the plot function place a grid of two plots using the space # defined by `ax` . fig, ax = plt.subplots(figsize=(12, 6)) ax.set_title("Decision Tree") -tree_disp = plot_partial_dependence(tree, X, ["LSTAT", "RM"], ax=ax) +tree_disp = plot_partial_dependence(tree, X, ["age", "bmi"], ax=ax) ############################################################################## # The partial depdendence curves can be plotted for the multi-layer perceptron. @@ -64,7 +63,7 @@ # the curve. fig, ax = plt.subplots(figsize=(12, 6)) ax.set_title("Multi-layer Perceptron") -mlp_disp = plot_partial_dependence(mlp, X, ["LSTAT", "RM"], ax=ax, +mlp_disp = plot_partial_dependence(mlp, X, ["age", "bmi"], ax=ax, line_kw={"c": "red"}) ############################################################################## @@ -124,14 +123,13 @@ tree_disp.axes_[0, 1].legend() plt.show() - ############################################################################## # Plotting partial dependence for one feature # =========================================== # -# Here, we plot the partial dependence curves for a single feature, "LSTAT", on +# Here, we plot the partial dependence curves for a single feature, "age", on # the same axes. In this case, `tree_disp.axes_` is passed into the second # plot function. -tree_disp = plot_partial_dependence(tree, X, ["LSTAT"]) -mlp_disp = plot_partial_dependence(mlp, X, ["LSTAT"], +tree_disp = plot_partial_dependence(tree, X, ["age"]) +mlp_disp = plot_partial_dependence(mlp, X, ["age"], ax=tree_disp.axes_, line_kw={"c": "red"}) diff --git a/examples/plot_roc_curve_visualization_api.py b/examples/plot_roc_curve_visualization_api.py index 55dec5649beeb..67592c12ec845 100644 --- a/examples/plot_roc_curve_visualization_api.py +++ b/examples/plot_roc_curve_visualization_api.py @@ -44,7 +44,7 @@ # We train a random forest classifier and create a plot comparing it to the SVC # ROC curve. Notice how `svc_disp` uses # :func:`~sklearn.metrics.RocCurveDisplay.plot` to plot the SVC ROC curve -# without recomputing the values of the roc curve itself. Futhermore, we +# without recomputing the values of the roc curve itself. Furthermore, we # pass `alpha=0.8` to the plot functions to adjust the alpha values of the # curves. rfc = RandomForestClassifier(n_estimators=10, random_state=42) diff --git a/examples/release_highlights/plot_release_highlights_0_22_0.py b/examples/release_highlights/plot_release_highlights_0_22_0.py index 6589f3f3351fc..450700d143ca2 100644 --- a/examples/release_highlights/plot_release_highlights_0_22_0.py +++ b/examples/release_highlights/plot_release_highlights_0_22_0.py @@ -26,7 +26,12 @@ # A new plotting API is available for creating visualizations. This new API # allows for quickly adjusting the visuals of a plot without involving any # recomputation. It is also possible to add different plots to the same -# figure. See more examples in the :ref:`User Guide `. +# figure. The following example illustrates :class:`~metrics.plot_roc_curve`, +# but other plots utilities are supported like +# :class:`~inspection.plot_partial_dependence`, +# :class:`~metrics.plot_precision_recall_curve`, and +# :class:`~metrics.plot_confusion_matrix`. Read more about this new API in the +# :ref:`User Guide `. from sklearn.model_selection import train_test_split from sklearn.svm import SVC @@ -241,11 +246,10 @@ def test_sklearn_compatible_estimator(estimator, check): # classification. Two averaging strategies are currently supported: the # one-vs-one algorithm computes the average of the pairwise ROC AUC scores, and # the one-vs-rest algorithm computes the average of the ROC AUC scores for each -# class against all other classes. In both cases, the predicted labels are -# provided in an array with values from 0 to ``n_classes``, and the scores -# correspond to the probability estimates that a sample belongs to a particular -# class. The OvO and OvR algorithms supports weighting uniformly -# (``average='macro'``) and weighting by the prevalence +# class against all other classes. In both cases, the multiclass ROC AUC scores +# are computed from the probability estimates that a sample belongs to a +# particular class according to the model. The OvO and OvR algorithms support +# weighting uniformly (``average='macro'``) and weighting by the prevalence # (``average='weighted'``). # # Read more in the :ref:`User Guide `. diff --git a/maint_tools/check_pxd_in_installation.py b/maint_tools/check_pxd_in_installation.py new file mode 100644 index 0000000000000..83c4b706294ad --- /dev/null +++ b/maint_tools/check_pxd_in_installation.py @@ -0,0 +1,59 @@ +"""Utility for testing presence and usability of .pxd files in the installation + +Usage: +------ +python check_pxd_in_installation.py path/to/install_dir/of/scikit-learn +""" + +import os +import sys +import pathlib +import tempfile +import textwrap +import subprocess + + +sklearn_dir = pathlib.Path(sys.argv[1]) +pxd_files = list(sklearn_dir.glob("**/*.pxd")) + +print("> Found pxd files:") +for pxd_file in pxd_files: + print(' -', pxd_file) + +print("\n> Trying to compile a cython extension cimporting all corresponding " + "modules\n") +with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + # A cython test file which cimports all modules corresponding to found + # pxd files. + # e.g. sklearn/tree/_utils.pxd becomes `cimport sklearn.tree._utils` + with open(tmpdir / 'tst.pyx', 'w') as f: + for pxd_file in pxd_files: + to_import = str(pxd_file.relative_to(sklearn_dir)) + to_import = to_import.replace(os.path.sep, '.') + to_import = to_import.replace('.pxd', '') + f.write('cimport sklearn.' + to_import + '\n') + + # A basic setup file to build the test file. + # We set the language to c++ and we use numpy.get_include() because + # some modules require it. + with open(tmpdir / 'setup_tst.py', 'w') as f: + f.write(textwrap.dedent( + """ + from distutils.core import setup + from distutils.extension import Extension + from Cython.Build import cythonize + import numpy + + extensions = [Extension("tst", + sources=["tst.pyx"], + language="c++", + include_dirs=[numpy.get_include()])] + + setup(ext_modules=cythonize(extensions)) + """)) + + subprocess.run(["python", "setup_tst.py", "build_ext", "-i"], + check=True, cwd=tmpdir) + + print("\n> Compilation succeeded !") diff --git a/maint_tools/test_docstrings.py b/maint_tools/test_docstrings.py index 6e1376fcab040..9c6c41f0b53dd 100644 --- a/maint_tools/test_docstrings.py +++ b/maint_tools/test_docstrings.py @@ -39,20 +39,36 @@ "SGDClassifier.score", "SGDClassifier.sparsify", "SGDClassifier.densify", + "VotingClassifier.fit", + "VotingClassifier.transform", + "VotingClassifier.predict", + "VotingClassifier.score", + "VotingClassifier.predict_proba", + "VotingClassifier.set_params", + "VotingClassifier.get_params", + "VotingClassifier.named_estimators", + "VotingClassifier$", ] def get_all_methods(): estimators = all_estimators() - for name, estimator in estimators: + for name, Estimator in estimators: if name.startswith("_"): # skip private classes continue - methods = [el for el in dir(estimator) if not el.startswith("_")] + methods = [] + for name in dir(Estimator): + if name.startswith("_"): + continue + method_obj = getattr(Estimator, name) + if (hasattr(method_obj, '__call__') + or isinstance(method_obj, property)): + methods.append(name) methods.append(None) for method in sorted(methods, key=lambda x: str(x)): - yield estimator, method + yield Estimator, method def filter_errors(errors, method): @@ -112,7 +128,16 @@ def repr_errors(res, estimator=None, method: Optional[str] = None) -> str: raise NotImplementedError if estimator is not None: - obj_signature = signature(getattr(estimator, method)) + obj = getattr(estimator, method) + try: + obj_signature = signature(obj) + except TypeError: + # In particular we can't parse the signature of properties + obj_signature = ( + "\nParsing of the method signature failed, " + "possibly because this is a property." + ) + obj_name = estimator.__name__ + "." + method else: obj_signature = "" @@ -120,7 +145,7 @@ def repr_errors(res, estimator=None, method: Optional[str] = None) -> str: msg = "\n\n" + "\n\n".join( [ - res["file"], + str(res["file"]), obj_name + str(obj_signature), res["docstring"], "# Errors", @@ -133,10 +158,10 @@ def repr_errors(res, estimator=None, method: Optional[str] = None) -> str: return msg -@pytest.mark.parametrize("estimator, method", get_all_methods()) -def test_docstring(estimator, method, request): - base_import_path = estimator.__module__ - import_path = [base_import_path, estimator.__name__] +@pytest.mark.parametrize("Estimator, method", get_all_methods()) +def test_docstring(Estimator, method, request): + base_import_path = Estimator.__module__ + import_path = [base_import_path, Estimator.__name__] if method is not None: import_path.append(method) @@ -154,7 +179,7 @@ def test_docstring(estimator, method, request): res["errors"] = list(filter_errors(res["errors"], method)) if res["errors"]: - msg = repr_errors(res, estimator, method) + msg = repr_errors(res, Estimator, method) raise ValueError(msg) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000..2547baae5874d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[build-system] +# Minimum requirements for the build system to execute. +requires = [ + "setuptools", + "wheel", + "Cython>=0.28.5", + "numpy>=1.13.3", + "scipy>=0.19.1", +] diff --git a/setup.cfg b/setup.cfg index 852310c1eeb23..f086993b26a29 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,16 +13,16 @@ addopts = --ignore maint_tools --doctest-modules --disable-pytest-warnings - -rs + -rxXs filterwarnings = ignore:the matrix subclass:PendingDeprecationWarning [wheelhouse_uploader] artifact_indexes= - # Wheels built by travis (only for specific tags): + # Wheels built by Azure Pipelines (only for specific tags): # https://github.com/MacPython/scikit-learn-wheels - http://wheels.scipy.org + https://pypi.anaconda.org/scikit-learn-wheels-staging/simple/scikit-learn/ [flake8] # Default flake8 3.5 ignored flags diff --git a/setup.py b/setup.py index 3ed5c786a17c3..90162b65644e5 100755 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ try: import builtins except ImportError: - # Python 2 compat: just to be able to declare that Python >=3.5 is needed. + # Python 2 compat: just to be able to declare that Python >=3.6 is needed. import __builtin__ as builtins # This is a bit (!) hackish: we are setting a global variable so that the @@ -52,10 +52,11 @@ SCIPY_MIN_VERSION = '1.1.0' NUMPY_MIN_VERSION = '1.14.0' else: - SCIPY_MIN_VERSION = '0.17.0' - NUMPY_MIN_VERSION = '1.11.0' + SCIPY_MIN_VERSION = '0.19.1' + NUMPY_MIN_VERSION = '1.13.3' JOBLIB_MIN_VERSION = '0.11' +THREADPOOLCTL_MIN_VERSION = '2.0.0' # Optional setuptools features # We need to import setuptools early, if we want setuptools features, @@ -139,7 +140,7 @@ def build_extensions(self): except ImportError: # Numpy should not be a dependency just to be able to introspect - # that python 3.5 is required. + # that python 3.6 is required. pass @@ -244,21 +245,23 @@ def setup_package(): 'Operating System :: Unix', 'Operating System :: MacOS', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', ('Programming Language :: Python :: ' 'Implementation :: CPython'), ('Programming Language :: Python :: ' 'Implementation :: PyPy') ], cmdclass=cmdclass, - python_requires=">=3.5", + python_requires=">=3.6", install_requires=[ 'numpy>={}'.format(NUMPY_MIN_VERSION), 'scipy>={}'.format(SCIPY_MIN_VERSION), - 'joblib>={}'.format(JOBLIB_MIN_VERSION) + 'joblib>={}'.format(JOBLIB_MIN_VERSION), + 'threadpoolctl>={}'.format(THREADPOOLCTL_MIN_VERSION) ], + package_data={'': ['*.pxd']}, **extra_setuptools_args) if len(sys.argv) == 1 or ( @@ -280,9 +283,9 @@ def setup_package(): metadata['version'] = VERSION else: - if sys.version_info < (3, 5): + if sys.version_info < (3, 6): raise RuntimeError( - "Scikit-learn requires Python 3.5 or later. The current" + "Scikit-learn requires Python 3.6 or later. The current" " Python version is %s installed in %s." % (platform.python_version(), sys.executable)) diff --git a/site.cfg b/site.cfg deleted file mode 100644 index 9055c7c25da37..0000000000000 --- a/site.cfg +++ /dev/null @@ -1,6 +0,0 @@ - -# Uncomment to link against the MKL library on windows -# [mkl] -# include_dirs=C:\Program Files\Intel\MKL\10.2.5.035\include -# library_dirs=C:\Program Files\Intel\MKL\10.2.5.035\ia32\lib -# mkl_libs=mkl_core, mkl_intel_c, mkl_intel_s, libguide, libguide40, mkl_blacs_dll, mkl_intel_sequential diff --git a/sklearn/__init__.py b/sklearn/__init__.py index 8b897eadc03e6..59aa672533524 100644 --- a/sklearn/__init__.py +++ b/sklearn/__init__.py @@ -40,7 +40,7 @@ # Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer. # 'X.Y.dev0' is the canonical version of 'X.Y.dev' # -__version__ = '0.22.dev0' +__version__ = '0.23.dev0' # On OSX, we can get a runtime error due to multiple OpenMP libraries loaded @@ -70,12 +70,18 @@ # We are not importing the rest of scikit-learn during the build # process, as it may not be compiled yet else: - from . import __check_build + # `_distributor_init` allows distributors to run custom init code. + # For instance, for the Windows wheel, this is used to pre-load the + # vcomp shared library runtime for OpenMP embedded in the sklearn/.libs + # sub-folder. + # It is necessary to do this prior to importing show_versions as the + # later is linked to the OpenMP runtime to make it possible to introspect + # it and importing it first would fail if the OpenMP dll cannot be found. + from . import _distributor_init # noqa: F401 + from . import __check_build # noqa: F401 from .base import clone from .utils._show_versions import show_versions - __check_build # avoid flakes unused variable error - __all__ = ['calibration', 'cluster', 'covariance', 'cross_decomposition', 'datasets', 'decomposition', 'dummy', 'ensemble', 'exceptions', 'experimental', 'externals', 'feature_extraction', @@ -100,7 +106,7 @@ def setup_module(module): # Check if a random seed exists in the environment, if not create one. _random_seed = os.environ.get('SKLEARN_SEED', None) if _random_seed is None: - _random_seed = np.random.uniform() * (2 ** 31 - 1) + _random_seed = np.random.uniform() * np.iinfo(np.int32).max _random_seed = int(_random_seed) print("I: Seeding RNGs with %r" % _random_seed) np.random.seed(_random_seed) diff --git a/sklearn/_build_utils/deprecated_modules.py b/sklearn/_build_utils/deprecated_modules.py index 9ff7c7f224710..045dc3d297be0 100644 --- a/sklearn/_build_utils/deprecated_modules.py +++ b/sklearn/_build_utils/deprecated_modules.py @@ -47,9 +47,9 @@ 'SpectralBiclustering'), ('_birch', 'sklearn.cluster.birch', 'sklearn.cluster', 'Birch'), ('_dbscan', 'sklearn.cluster.dbscan_', 'sklearn.cluster', 'DBSCAN'), - ('_hierarchical', 'sklearn.cluster.hierarchical', 'sklearn.cluster', + ('_agglomerative', 'sklearn.cluster.hierarchical', 'sklearn.cluster', 'FeatureAgglomeration'), - ('_k_means', 'sklearn.cluster.k_means_', 'sklearn.cluster', 'KMeans'), + ('_kmeans', 'sklearn.cluster.k_means_', 'sklearn.cluster', 'KMeans'), ('_mean_shift', 'sklearn.cluster.mean_shift_', 'sklearn.cluster', 'MeanShift'), ('_optics', 'sklearn.cluster.optics_', 'sklearn.cluster', 'OPTICS'), @@ -101,7 +101,7 @@ ('_kernel_pca', 'sklearn.decomposition.kernel_pca', 'sklearn.decomposition', 'KernelPCA'), ('_nmf', 'sklearn.decomposition.nmf', 'sklearn.decomposition', 'NMF'), - ('_online_lda', 'sklearn.decomposition.online_lda', + ('_lda', 'sklearn.decomposition.online_lda', 'sklearn.decomposition', 'LatentDirichletAllocation'), ('_online_lda_fast', 'sklearn.decomposition.online_lda_fast', 'sklearn.decomposition', 'mean_change'), @@ -133,14 +133,14 @@ 'sklearn.datasets', 'make_classification'), ('_species_distributions', 'sklearn.datasets.species_distributions', 'sklearn.datasets', 'fetch_species_distributions'), - ('_svmlight_format', 'sklearn.datasets.svmlight_format', + ('_svmlight_format_io', 'sklearn.datasets.svmlight_format', 'sklearn.datasets', 'load_svmlight_file'), ('_twenty_newsgroups', 'sklearn.datasets.twenty_newsgroups', 'sklearn.datasets', 'strip_newsgroup_header'), ('_dict_vectorizer', 'sklearn.feature_extraction.dict_vectorizer', 'sklearn.feature_extraction', 'DictVectorizer'), - ('_hashing', 'sklearn.feature_extraction.hashing', + ('_hash', 'sklearn.feature_extraction.hashing', 'sklearn.feature_extraction', 'FeatureHasher'), ('_stop_words', 'sklearn.feature_extraction.stop_words', 'sklearn.feature_extraction.text', 'ENGLISH_STOP_WORDS'), diff --git a/sklearn/_distributor_init.py b/sklearn/_distributor_init.py new file mode 100644 index 0000000000000..a0142ac80878f --- /dev/null +++ b/sklearn/_distributor_init.py @@ -0,0 +1,10 @@ +""" Distributor init file + +Distributors: you can add custom code here to support particular distributions +of scikit-learn. + +For example, this is a good place to put any checks for hardware requirements. + +The scikit-learn standard source distribution will not put code in this file, +so you can safely replace this file with your own version. +""" diff --git a/sklearn/_loss/__init__.py b/sklearn/_loss/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/_loss/glm_distribution.py b/sklearn/_loss/glm_distribution.py new file mode 100644 index 0000000000000..cb20fda1c022d --- /dev/null +++ b/sklearn/_loss/glm_distribution.py @@ -0,0 +1,355 @@ +""" +Distribution functions used in GLM +""" + +# Author: Christian Lorentzen +# License: BSD 3 clause + +from abc import ABCMeta, abstractmethod +from collections import namedtuple +import numbers + +import numpy as np +from scipy.special import xlogy + + +DistributionBoundary = namedtuple("DistributionBoundary", + ("value", "inclusive")) + + +class ExponentialDispersionModel(metaclass=ABCMeta): + r"""Base class for reproductive Exponential Dispersion Models (EDM). + + The pdf of :math:`Y\sim \mathrm{EDM}(y_\textrm{pred}, \phi)` is given by + + .. math:: p(y| \theta, \phi) = c(y, \phi) + \exp\left(\frac{\theta y-A(\theta)}{\phi}\right) + = \tilde{c}(y, \phi) + \exp\left(-\frac{d(y, y_\textrm{pred})}{2\phi}\right) + + with mean :math:`\mathrm{E}[Y] = A'(\theta) = y_\textrm{pred}`, + variance :math:`\mathrm{Var}[Y] = \phi \cdot v(y_\textrm{pred})`, + unit variance :math:`v(y_\textrm{pred})` and + unit deviance :math:`d(y,y_\textrm{pred})`. + + Methods + ------- + deviance + deviance_derivative + in_y_range + unit_deviance + unit_deviance_derivative + unit_variance + + References + ---------- + https://en.wikipedia.org/wiki/Exponential_dispersion_model. + """ + + def in_y_range(self, y): + """Returns ``True`` if y is in the valid range of Y~EDM. + + Parameters + ---------- + y : array of shape (n_samples,) + Target values. + """ + # Note that currently supported distributions have +inf upper bound + + if not isinstance(self._lower_bound, DistributionBoundary): + raise TypeError('_lower_bound attribute must be of type ' + 'DistributionBoundary') + + if self._lower_bound.inclusive: + return np.greater_equal(y, self._lower_bound.value) + else: + return np.greater(y, self._lower_bound.value) + + @abstractmethod + def unit_variance(self, y_pred): + r"""Compute the unit variance function. + + The unit variance :math:`v(y_\textrm{pred})` determines the variance as + a function of the mean :math:`y_\textrm{pred}` by + :math:`\mathrm{Var}[Y_i] = \phi/s_i*v(y_\textrm{pred}_i)`. + It can also be derived from the unit deviance + :math:`d(y,y_\textrm{pred})` as + + .. math:: v(y_\textrm{pred}) = \frac{2}{ + \frac{\partial^2 d(y,y_\textrm{pred})}{ + \partialy_\textrm{pred}^2}}\big|_{y=y_\textrm{pred}} + + See also :func:`variance`. + + Parameters + ---------- + y_pred : array of shape (n_samples,) + Predicted mean. + """ + + @abstractmethod + def unit_deviance(self, y, y_pred, check_input=False): + r"""Compute the unit deviance. + + The unit_deviance :math:`d(y,y_\textrm{pred})` can be defined by the + log-likelihood as + :math:`d(y,y_\textrm{pred}) = -2\phi\cdot + \left(loglike(y,y_\textrm{pred},\phi) - loglike(y,y,\phi)\right).` + + Parameters + ---------- + y : array of shape (n_samples,) + Target values. + + y_pred : array of shape (n_samples,) + Predicted mean. + + check_input : bool, default=False + If True raise an exception on invalid y or y_pred values, otherwise + they will be propagated as NaN. + Returns + ------- + deviance: array of shape (n_samples,) + Computed deviance + """ + + def unit_deviance_derivative(self, y, y_pred): + r"""Compute the derivative of the unit deviance w.r.t. y_pred. + + The derivative of the unit deviance is given by + :math:`\frac{\partial}{\partialy_\textrm{pred}}d(y,y_\textrm{pred}) + = -2\frac{y-y_\textrm{pred}}{v(y_\textrm{pred})}` + with unit variance :math:`v(y_\textrm{pred})`. + + Parameters + ---------- + y : array of shape (n_samples,) + Target values. + + y_pred : array of shape (n_samples,) + Predicted mean. + """ + return -2 * (y - y_pred) / self.unit_variance(y_pred) + + def deviance(self, y, y_pred, weights=1): + r"""Compute the deviance. + + The deviance is a weighted sum of the per sample unit deviances, + :math:`D = \sum_i s_i \cdot d(y_i, y_\textrm{pred}_i)` + with weights :math:`s_i` and unit deviance + :math:`d(y,y_\textrm{pred})`. + In terms of the log-likelihood it is :math:`D = -2\phi\cdot + \left(loglike(y,y_\textrm{pred},\frac{phi}{s}) + - loglike(y,y,\frac{phi}{s})\right)`. + + Parameters + ---------- + y : array of shape (n_samples,) + Target values. + + y_pred : array of shape (n_samples,) + Predicted mean. + + weights : {int, array of shape (n_samples,)}, default=1 + Weights or exposure to which variance is inverse proportional. + """ + return np.sum(weights * self.unit_deviance(y, y_pred)) + + def deviance_derivative(self, y, y_pred, weights=1): + r"""Compute the derivative of the deviance w.r.t. y_pred. + + It gives :math:`\frac{\partial}{\partial y_\textrm{pred}} + D(y, \y_\textrm{pred}; weights)`. + + Parameters + ---------- + y : array, shape (n_samples,) + Target values. + + y_pred : array, shape (n_samples,) + Predicted mean. + + weights : {int, array of shape (n_samples,)}, default=1 + Weights or exposure to which variance is inverse proportional. + """ + return weights * self.unit_deviance_derivative(y, y_pred) + + +class TweedieDistribution(ExponentialDispersionModel): + r"""A class for the Tweedie distribution. + + A Tweedie distribution with mean :math:`y_\textrm{pred}=\mathrm{E}[Y]` + is uniquely defined by it's mean-variance relationship + :math:`\mathrm{Var}[Y] \propto y_\textrm{pred}^power`. + + Special cases are: + + ===== ================ + Power Distribution + ===== ================ + 0 Normal + 1 Poisson + (1,2) Compound Poisson + 2 Gamma + 3 Inverse Gaussian + + Parameters + ---------- + power : float, default=0 + The variance power of the `unit_variance` + :math:`v(y_\textrm{pred}) = y_\textrm{pred}^{power}`. + For ``0=1.') + elif 1 <= power < 2: + # Poisson or Compound Poisson distribution + self._lower_bound = DistributionBoundary(0, inclusive=True) + elif power >= 2: + # Gamma, Positive Stable, Inverse Gaussian distributions + self._lower_bound = DistributionBoundary(0, inclusive=False) + else: # pragma: no cover + # this branch should be unreachable. + raise ValueError + + self._power = power + + def unit_variance(self, y_pred): + """Compute the unit variance of a Tweedie distribution + v(y_\textrm{pred})=y_\textrm{pred}**power. + + Parameters + ---------- + y_pred : array of shape (n_samples,) + Predicted mean. + """ + return np.power(y_pred, self.power) + + def unit_deviance(self, y, y_pred, check_input=False): + r"""Compute the unit deviance. + + The unit_deviance :math:`d(y,y_\textrm{pred})` can be defined by the + log-likelihood as + :math:`d(y,y_\textrm{pred}) = -2\phi\cdot + \left(loglike(y,y_\textrm{pred},\phi) - loglike(y,y,\phi)\right).` + + Parameters + ---------- + y : array of shape (n_samples,) + Target values. + + y_pred : array of shape (n_samples,) + Predicted mean. + + check_input : bool, default=False + If True raise an exception on invalid y or y_pred values, otherwise + they will be propagated as NaN. + Returns + ------- + deviance: array of shape (n_samples,) + Computed deviance + """ + p = self.power + + if check_input: + message = ("Mean Tweedie deviance error with power={} can only be " + "used on ".format(p)) + if p < 0: + # 'Extreme stable', y any realy number, y_pred > 0 + if (y_pred <= 0).any(): + raise ValueError(message + "strictly positive y_pred.") + elif p == 0: + # Normal, y and y_pred can be any real number + pass + elif 0 < p < 1: + raise ValueError("Tweedie deviance is only defined for " + "power<=0 and power>=1.") + elif 1 <= p < 2: + # Poisson and Compount poisson distribution, y >= 0, y_pred > 0 + if (y < 0).any() or (y_pred <= 0).any(): + raise ValueError(message + "non-negative y and strictly " + "positive y_pred.") + elif p >= 2: + # Gamma and Extreme stable distribution, y and y_pred > 0 + if (y <= 0).any() or (y_pred <= 0).any(): + raise ValueError(message + + "strictly positive y and y_pred.") + else: # pragma: nocover + # Unreachable statement + raise ValueError + + if p < 0: + # 'Extreme stable', y any realy number, y_pred > 0 + dev = 2 * (np.power(np.maximum(y, 0), 2-p) / ((1-p) * (2-p)) + - y * np.power(y_pred, 1-p) / (1-p) + + np.power(y_pred, 2-p) / (2-p)) + + elif p == 0: + # Normal distribution, y and y_pred any real number + dev = (y - y_pred)**2 + elif p < 1: + raise ValueError("Tweedie deviance is only defined for power<=0 " + "and power>=1.") + elif p == 1: + # Poisson distribution + dev = 2 * (xlogy(y, y/y_pred) - y + y_pred) + elif p == 2: + # Gamma distribution + dev = 2 * (np.log(y_pred/y) + y/y_pred - 1) + else: + dev = 2 * (np.power(y, 2-p) / ((1-p) * (2-p)) + - y * np.power(y_pred, 1-p) / (1-p) + + np.power(y_pred, 2-p) / (2-p)) + return dev + + +class NormalDistribution(TweedieDistribution): + """Class for the Normal (aka Gaussian) distribution""" + def __init__(self): + super().__init__(power=0) + + +class PoissonDistribution(TweedieDistribution): + """Class for the scaled Poisson distribution""" + def __init__(self): + super().__init__(power=1) + + +class GammaDistribution(TweedieDistribution): + """Class for the Gamma distribution""" + def __init__(self): + super().__init__(power=2) + + +class InverseGaussianDistribution(TweedieDistribution): + """Class for the scaled InverseGaussianDistribution distribution""" + def __init__(self): + super().__init__(power=3) + + +EDM_DISTRIBUTIONS = { + 'normal': NormalDistribution, + 'poisson': PoissonDistribution, + 'gamma': GammaDistribution, + 'inverse-gaussian': InverseGaussianDistribution, +} diff --git a/sklearn/_loss/tests/__init__.py b/sklearn/_loss/tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/_loss/tests/test_glm_distribution.py b/sklearn/_loss/tests/test_glm_distribution.py new file mode 100644 index 0000000000000..cb4c5ae07e4d1 --- /dev/null +++ b/sklearn/_loss/tests/test_glm_distribution.py @@ -0,0 +1,112 @@ +# Authors: Christian Lorentzen +# +# License: BSD 3 clause +import numpy as np +from numpy.testing import ( + assert_allclose, + assert_array_equal, +) +from scipy.optimize import check_grad +import pytest + +from sklearn._loss.glm_distribution import ( + TweedieDistribution, + NormalDistribution, PoissonDistribution, + GammaDistribution, InverseGaussianDistribution, + DistributionBoundary +) + + +@pytest.mark.parametrize( + 'family, expected', + [(NormalDistribution(), [True, True, True]), + (PoissonDistribution(), [False, True, True]), + (TweedieDistribution(power=1.5), [False, True, True]), + (GammaDistribution(), [False, False, True]), + (InverseGaussianDistribution(), [False, False, True]), + (TweedieDistribution(power=4.5), [False, False, True])]) +def test_family_bounds(family, expected): + """Test the valid range of distributions at -1, 0, 1.""" + result = family.in_y_range([-1, 0, 1]) + assert_array_equal(result, expected) + + +def test_invalid_distribution_bound(): + dist = TweedieDistribution() + dist._lower_bound = 0 + with pytest.raises(TypeError, + match="must be of type DistributionBoundary"): + dist.in_y_range([-1, 0, 1]) + + +def test_tweedie_distribution_power(): + msg = "distribution is only defined for power<=0 and power>=1" + with pytest.raises(ValueError, match=msg): + TweedieDistribution(power=0.5) + + with pytest.raises(TypeError, match="must be a real number"): + TweedieDistribution(power=1j) + + with pytest.raises(TypeError, match="must be a real number"): + dist = TweedieDistribution() + dist.power = 1j + + dist = TweedieDistribution() + assert isinstance(dist._lower_bound, DistributionBoundary) + + assert dist._lower_bound.inclusive is False + dist.power = 1 + assert dist._lower_bound.value == 0.0 + assert dist._lower_bound.inclusive is True + + +@pytest.mark.parametrize( + 'family, chk_values', + [(NormalDistribution(), [-1.5, -0.1, 0.1, 2.5]), + (PoissonDistribution(), [0.1, 1.5]), + (GammaDistribution(), [0.1, 1.5]), + (InverseGaussianDistribution(), [0.1, 1.5]), + (TweedieDistribution(power=-2.5), [0.1, 1.5]), + (TweedieDistribution(power=-1), [0.1, 1.5]), + (TweedieDistribution(power=1.5), [0.1, 1.5]), + (TweedieDistribution(power=2.5), [0.1, 1.5]), + (TweedieDistribution(power=-4), [0.1, 1.5])]) +def test_deviance_zero(family, chk_values): + """Test deviance(y,y) = 0 for different families.""" + for x in chk_values: + assert_allclose(family.deviance(x, x), 0, atol=1e-9) + + +@pytest.mark.parametrize( + 'family', + [NormalDistribution(), + PoissonDistribution(), + GammaDistribution(), + InverseGaussianDistribution(), + TweedieDistribution(power=-2.5), + TweedieDistribution(power=-1), + TweedieDistribution(power=1.5), + TweedieDistribution(power=2.5), + TweedieDistribution(power=-4)], + ids=lambda x: x.__class__.__name__ +) +def test_deviance_derivative(family): + """Test deviance derivative for different families.""" + rng = np.random.RandomState(0) + y_true = rng.rand(10) + # make data positive + y_true += np.abs(y_true.min()) + 1e-2 + + y_pred = y_true + np.fmax(rng.rand(10), 0.) + + dev = family.deviance(y_true, y_pred) + assert isinstance(dev, float) + dev_derivative = family.deviance_derivative(y_true, y_pred) + assert dev_derivative.shape == y_pred.shape + + err = check_grad( + lambda y_pred: family.deviance(y_true, y_pred), + lambda y_pred: family.deviance_derivative(y_true, y_pred), + y_pred, + ) / np.linalg.norm(dev_derivative) + assert abs(err) < 1e-6 diff --git a/sklearn/base.py b/sklearn/base.py index 7ededd7a70548..70dec8c030418 100644 --- a/sklearn/base.py +++ b/sklearn/base.py @@ -1,4 +1,8 @@ -"""Base classes for all estimators.""" +""" +Base classes for all estimators. + +Used for VotingClassifier +""" # Author: Gael Varoquaux # License: BSD 3 clause @@ -14,6 +18,8 @@ from . import __version__ from .utils import _IS_32BIT +from .utils.validation import check_X_y +from .utils.validation import check_array _DEFAULT_TAGS = { 'non_deterministic': False, @@ -27,6 +33,7 @@ 'stateless': False, 'multilabel': False, '_skip_test': False, + '_xfail_test': False, 'multioutput_only': False, 'binary_only': False, 'requires_fit': True} @@ -44,7 +51,7 @@ def clone(estimator, safe=True): estimator : estimator object, or list, tuple or set of objects The estimator or group of estimators to be cloned - safe : boolean, optional + safe : bool, default=True If safe is false, clone will fall back to a deep copy on objects that are not estimators. @@ -57,10 +64,17 @@ def clone(estimator, safe=True): if not safe: return copy.deepcopy(estimator) else: - raise TypeError("Cannot clone object '%s' (type %s): " - "it does not seem to be a scikit-learn estimator " - "as it does not implement a 'get_params' methods." - % (repr(estimator), type(estimator))) + if isinstance(estimator, type): + raise TypeError("Cannot clone object. " + + "You should provide an instance of " + + "scikit-learn estimator instead of a class.") + else: + raise TypeError("Cannot clone object '%s' (type %s): " + "it does not seem to be a scikit-learn " + "estimator as it does not implement a " + "'get_params' method." + % (repr(estimator), type(estimator))) + klass = estimator.__class__ new_object_params = estimator.get_params(deep=False) for name, param in new_object_params.items(): @@ -87,10 +101,10 @@ def _pprint(params, offset=0, printer=repr): params : dict The dictionary to pretty print - offset : int + offset : int, default=0 The offset in characters to add at the begin of each line. - printer : callable + printer : callable, default=repr The function to convert entries to strings, typically the builtin str or repr @@ -331,9 +345,76 @@ def _get_tags(self): collected_tags.update(more_tags) return collected_tags + def _check_n_features(self, X, reset): + """Set the `n_features_in_` attribute, or check against it. + + Parameters + ---------- + X : {ndarray, sparse matrix} of shape (n_samples, n_features) + The input samples. + reset : bool + If True, the `n_features_in_` attribute is set to `X.shape[1]`. + Else, the attribute must already exist and the function checks + that it is equal to `X.shape[1]`. + """ + n_features = X.shape[1] + + if reset: + self.n_features_in_ = n_features + else: + if not hasattr(self, 'n_features_in_'): + raise RuntimeError( + "The reset parameter is False but there is no " + "n_features_in_ attribute. Is this estimator fitted?" + ) + if n_features != self.n_features_in_: + raise ValueError( + 'X has {} features, but this {} is expecting {} features ' + 'as input.'.format(n_features, self.__class__.__name__, + self.n_features_in_) + ) + + def _validate_data(self, X, y=None, reset=True, **check_params): + """Validate input data and set or check the `n_features_in_` attribute. + + Parameters + ---------- + X : {array-like, sparse matrix, dataframe} of shape \ + (n_samples, n_features) + The input samples. + y : array-like of shape (n_samples,), default=None + The targets. If None, `check_array` is called on `X` and + `check_X_y` is called otherwise. + reset : bool, default=True + Whether to reset the `n_features_in_` attribute. + If False, the input will be checked for consistency with data + provided when reset was last True. + **check_params : kwargs + Parameters passed to :func:`sklearn.utils.check_array` or + :func:`sklearn.utils.check_X_y`. + + Returns + ------- + out : {ndarray, sparse matrix} or tuple of these + The validated input. A tuple is returned if `y` is not None. + """ + + if y is None: + X = check_array(X, **check_params) + out = X + else: + X, y = check_X_y(X, y, **check_params) + out = X, y + + if check_params.get('ensure_2d', True): + self._check_n_features(X, reset=reset) + + return out + class ClassifierMixin: """Mixin class for all classifiers in scikit-learn.""" + _estimator_type = "classifier" def score(self, X, y, sample_weight=None): @@ -383,8 +464,9 @@ def score(self, X, y, sample_weight=None): ---------- X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a - precomputed kernel matrix instead, shape = (n_samples, - n_samples_fitted], where n_samples_fitted is the number of + precomputed kernel matrix or a list of generic objects instead, + shape = (n_samples, n_samples_fitted), + where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) @@ -400,34 +482,17 @@ def score(self, X, y, sample_weight=None): Notes ----- - The R2 score used when calling ``score`` on a regressor will use + The R2 score used when calling ``score`` on a regressor uses ``multioutput='uniform_average'`` from version 0.23 to keep consistent - with :func:`~sklearn.metrics.r2_score`. This will influence the - ``score`` method of all the multioutput regressors (except for - :class:`~sklearn.multioutput.MultiOutputRegressor`). To specify the - default value manually and avoid the warning, please either call - :func:`~sklearn.metrics.r2_score` directly or make a custom scorer with - :func:`~sklearn.metrics.make_scorer` (the built-in scorer ``'r2'`` uses - ``multioutput='uniform_average'``). + with default value of :func:`~sklearn.metrics.r2_score`. + This influences the ``score`` method of all the multioutput + regressors (except for + :class:`~sklearn.multioutput.MultiOutputRegressor`). """ from .metrics import r2_score - from .metrics._regression import _check_reg_targets y_pred = self.predict(X) - # XXX: Remove the check in 0.23 - y_type, _, _, _ = _check_reg_targets(y, y_pred, None) - if y_type == 'continuous-multioutput': - warnings.warn("The default value of multioutput (not exposed in " - "score method) will change from 'variance_weighted' " - "to 'uniform_average' in 0.23 to keep consistent " - "with 'metrics.r2_score'. To specify the default " - "value manually and avoid the warning, please " - "either call 'metrics.r2_score' directly or make a " - "custom scorer with 'metrics.make_scorer' (the " - "built-in scorer 'r2' uses " - "multioutput='uniform_average').", FutureWarning) - return r2_score(y, y_pred, sample_weight=sample_weight, - multioutput='variance_weighted') + return r2_score(y, y_pred, sample_weight=sample_weight) class ClusterMixin: @@ -440,7 +505,7 @@ def fit_predict(self, X, y=None): Parameters ---------- - X : ndarray, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Input data. y : Ignored @@ -448,7 +513,7 @@ def fit_predict(self, X, y=None): Returns ------- - labels : ndarray, shape (n_samples,) + labels : ndarray of shape (n_samples,) Cluster labels. """ # non-optimized default implementation; override when a better @@ -480,9 +545,9 @@ def get_indices(self, i): Returns ------- - row_ind : np.array, dtype=np.intp + row_ind : ndarray, dtype=np.intp Indices of rows in the dataset that belong to the bicluster. - col_ind : np.array, dtype=np.intp + col_ind : ndarray, dtype=np.intp Indices of columns in the dataset that belong to the bicluster. """ @@ -513,12 +578,12 @@ def get_submatrix(self, i, data): ---------- i : int The index of the cluster. - data : array + data : array-like The data. Returns ------- - submatrix : array + submatrix : ndarray The submatrix corresponding to bicluster i. Notes @@ -544,10 +609,10 @@ def fit_transform(self, X, y=None, **fit_params): Parameters ---------- - X : numpy array of shape [n_samples, n_features] + X : ndarray of shape (n_samples, n_features) Training set. - y : numpy array of shape [n_samples] + y : ndarray of shape (n_samples,), default=None Target values. **fit_params : dict @@ -555,7 +620,7 @@ def fit_transform(self, X, y=None, **fit_params): Returns ------- - X_new : numpy array of shape [n_samples, n_features_new] + X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. """ # non-optimized default implementation; override when a better @@ -579,6 +644,9 @@ def score(self, X, y=None): ---------- X : array-like of shape (n_samples, n_features) + y : Ignored + Not used, present for API consistency by convention. + Returns ------- score : float @@ -597,7 +665,7 @@ def fit_predict(self, X, y=None): Parameters ---------- - X : ndarray, shape (n_samples, n_features) + X : ndarray of shape (n_samples, n_features) Input data. y : Ignored @@ -605,7 +673,7 @@ def fit_predict(self, X, y=None): Returns ------- - y : ndarray, shape (n_samples,) + y : ndarray of shape (n_samples,) 1 for inliers, -1 for outliers. """ # override for transductive outlier detectors like LocalOulierFactor diff --git a/sklearn/calibration.py b/sklearn/calibration.py index 4c8a81a2137ec..a5490efa28c0a 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -27,38 +27,30 @@ from .isotonic import IsotonicRegression from .svm import LinearSVC from .model_selection import check_cv +from .utils.validation import _deprecate_positional_args class CalibratedClassifierCV(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): - """Probability calibration with isotonic regression or sigmoid. - - See glossary entry for :term:`cross-validation estimator`. + """Probability calibration with isotonic regression or logistic regression. - With this class, the base_estimator is fit on the train set of the - cross-validation generator and the test set is used for calibration. - The probabilities for each of the folds are then averaged - for prediction. In case that cv="prefit" is passed to __init__, - it is assumed that base_estimator has been fitted already and all - data is used for calibration. Note that data for fitting the - classifier and for calibrating it must be disjoint. + The calibration is based on the :term:`decision_function` method of the + `base_estimator` if it exists, else on :term:`predict_proba`. Read more in the :ref:`User Guide `. Parameters ---------- base_estimator : instance BaseEstimator - The classifier whose output decision function needs to be calibrated - to offer more accurate predict_proba outputs. If cv=prefit, the - classifier must have been fit already on data. + The classifier whose output need to be calibrated to provide more + accurate `predict_proba` outputs. method : 'sigmoid' or 'isotonic' The method to use for calibration. Can be 'sigmoid' which - corresponds to Platt's method or 'isotonic' which is a - non-parametric approach. It is not advised to use isotonic calibration - with too few calibration samples ``(<<1000)`` since it tends to - overfit. - Use sigmoids (Platt's calibration) in this case. + corresponds to Platt's method (i.e. a logistic regression model) or + 'isotonic' which is a non-parametric approach. It is not advised to + use isotonic calibration with too few calibration samples + ``(<<1000)`` since it tends to overfit. cv : integer, cross-validation generator, iterable or "prefit", optional Determines the cross-validation splitting strategy. @@ -77,7 +69,7 @@ class CalibratedClassifierCV(BaseEstimator, ClassifierMixin, Refer :ref:`User Guide ` for the various cross-validation strategies that can be used here. - If "prefit" is passed, it is assumed that base_estimator has been + If "prefit" is passed, it is assumed that `base_estimator` has been fitted already and all data is used for calibration. .. versionchanged:: 0.22 @@ -89,7 +81,7 @@ class CalibratedClassifierCV(BaseEstimator, ClassifierMixin, The class labels. calibrated_classifiers_ : list (len() equal to cv or 1 if cv == "prefit") - The list of calibrated classifiers, one for each crossvalidation fold, + The list of calibrated classifiers, one for each cross-validation fold, which has been fitted on all but the validation fold and calibrated on the validation fold. @@ -107,7 +99,8 @@ class CalibratedClassifierCV(BaseEstimator, ClassifierMixin, .. [4] Predicting Good Probabilities with Supervised Learning, A. Niculescu-Mizil & R. Caruana, ICML 2005 """ - def __init__(self, base_estimator=None, method='sigmoid', cv=None): + @_deprecate_positional_args + def __init__(self, base_estimator=None, *, method='sigmoid', cv=None): self.base_estimator = base_estimator self.method = method self.cv = cv @@ -131,8 +124,8 @@ def fit(self, X, y, sample_weight=None): self : object Returns an instance of self. """ - X, y = check_X_y(X, y, accept_sparse=['csc', 'csr', 'coo'], - force_all_finite=False, allow_nd=True) + X, y = self._validate_data(X, y, accept_sparse=['csc', 'csr', 'coo'], + force_all_finite=False, allow_nd=True) X, y = indexable(X, y) le = LabelBinarizer().fit(y) self.classes_ = le.classes_ @@ -159,43 +152,35 @@ def fit(self, X, y, sample_weight=None): if self.cv == "prefit": calibrated_classifier = _CalibratedClassifier( base_estimator, method=self.method) - if sample_weight is not None: - calibrated_classifier.fit(X, y, sample_weight) - else: - calibrated_classifier.fit(X, y) + calibrated_classifier.fit(X, y, sample_weight) self.calibrated_classifiers_.append(calibrated_classifier) else: cv = check_cv(self.cv, y, classifier=True) fit_parameters = signature(base_estimator.fit).parameters - estimator_name = type(base_estimator).__name__ - if (sample_weight is not None - and "sample_weight" not in fit_parameters): - warnings.warn("%s does not support sample_weight. Samples" - " weights are only used for the calibration" - " itself." % estimator_name) - sample_weight = check_array(sample_weight, ensure_2d=False) - base_estimator_sample_weight = None - else: - if sample_weight is not None: - sample_weight = _check_sample_weight(sample_weight, X) - base_estimator_sample_weight = sample_weight + base_estimator_supports_sw = "sample_weight" in fit_parameters + + if sample_weight is not None: + sample_weight = _check_sample_weight(sample_weight, X) + + if not base_estimator_supports_sw: + estimator_name = type(base_estimator).__name__ + warnings.warn("Since %s does not support sample_weights, " + "sample weights will only be used for the " + "calibration itself." % estimator_name) + for train, test in cv.split(X, y): this_estimator = clone(base_estimator) - if base_estimator_sample_weight is not None: - this_estimator.fit( - X[train], y[train], - sample_weight=base_estimator_sample_weight[train]) + + if sample_weight is not None and base_estimator_supports_sw: + this_estimator.fit(X[train], y[train], + sample_weight=sample_weight[train]) else: this_estimator.fit(X[train], y[train]) calibrated_classifier = _CalibratedClassifier( - this_estimator, method=self.method, - classes=self.classes_) - if sample_weight is not None: - calibrated_classifier.fit(X[test], y[test], - sample_weight[test]) - else: - calibrated_classifier.fit(X[test], y[test]) + this_estimator, method=self.method, classes=self.classes_) + sw = None if sample_weight is None else sample_weight[test] + calibrated_classifier.fit(X[test], y[test], sample_weight=sw) self.calibrated_classifiers_.append(calibrated_classifier) return self @@ -231,8 +216,9 @@ def predict_proba(self, X): return mean_proba def predict(self, X): - """Predict the target of new samples. Can be different from the - prediction of the uncalibrated classifier. + """Predict the target of new samples. The predicted class is the + class that has the highest probability, and can thus be different + from the prediction of the uncalibrated classifier. Parameters ---------- @@ -291,7 +277,8 @@ class _CalibratedClassifier: .. [4] Predicting Good Probabilities with Supervised Learning, A. Niculescu-Mizil & R. Caruana, ICML 2005 """ - def __init__(self, base_estimator, method='sigmoid', classes=None): + @_deprecate_positional_args + def __init__(self, base_estimator, *, method='sigmoid', classes=None): self.base_estimator = base_estimator self.method = method self.classes = classes @@ -523,7 +510,8 @@ def calibration_curve(y_true, y_prob, normalize=False, n_bins=5, strategy='uniform'): """Compute true and predicted probabilities for a calibration curve. - The method assumes the inputs come from a binary classifier. + The method assumes the inputs come from a binary classifier, and + discretize the [0, 1] interval into bins. Calibration curves may also be referred to as reliability diagrams. @@ -531,36 +519,38 @@ def calibration_curve(y_true, y_prob, normalize=False, n_bins=5, Parameters ---------- - y_true : array, shape (n_samples,) + y_true : array-like of shape (n_samples,) True targets. - y_prob : array, shape (n_samples,) + y_prob : array-like of shape (n_samples,) Probabilities of the positive class. - normalize : bool, optional, default=False - Whether y_prob needs to be normalized into the bin [0, 1], i.e. is not - a proper probability. If True, the smallest value in y_prob is mapped - onto 0 and the largest one onto 1. + normalize : bool, default=False + Whether y_prob needs to be normalized into the [0, 1] interval, i.e. + is not a proper probability. If True, the smallest value in y_prob + is linearly mapped onto 0 and the largest one onto 1. - n_bins : int - Number of bins. A bigger number requires more data. Bins with no data - points (i.e. without corresponding values in y_prob) will not be - returned, thus there may be fewer than n_bins in the return value. + n_bins : int, default=5 + Number of bins to discretize the [0, 1] interval. A bigger number + requires more data. Bins with no samples (i.e. without + corresponding values in `y_prob`) will not be returned, thus the + returned arrays may have less than `n_bins` values. - strategy : {'uniform', 'quantile'}, (default='uniform') + strategy : {'uniform', 'quantile'}, default='uniform' Strategy used to define the widths of the bins. uniform - All bins have identical widths. + The bins have identical widths. quantile - All bins have the same number of points. + The bins have the same number of samples and depend on `y_prob`. Returns ------- - prob_true : array, shape (n_bins,) or smaller - The true probability in each bin (fraction of positives). + prob_true : ndarray of shape (n_bins,) or smaller + The proportion of samples whose class is the positive class, in each + bin (fraction of positives). - prob_pred : array, shape (n_bins,) or smaller + prob_pred : ndarray of shape (n_bins,) or smaller The mean predicted probability in each bin. References @@ -603,7 +593,7 @@ def calibration_curve(y_true, y_prob, normalize=False, n_bins=5, bin_total = np.bincount(binids, minlength=len(bins)) nonzero = bin_total != 0 - prob_true = (bin_true[nonzero] / bin_total[nonzero]) - prob_pred = (bin_sums[nonzero] / bin_total[nonzero]) + prob_true = bin_true[nonzero] / bin_total[nonzero] + prob_pred = bin_sums[nonzero] / bin_total[nonzero] return prob_true, prob_pred diff --git a/sklearn/cluster/__init__.py b/sklearn/cluster/__init__.py index 2cdf4b074e1c3..5f3cc58507576 100644 --- a/sklearn/cluster/__init__.py +++ b/sklearn/cluster/__init__.py @@ -7,9 +7,9 @@ from ._mean_shift import (mean_shift, MeanShift, estimate_bandwidth, get_bin_seeds) from ._affinity_propagation import affinity_propagation, AffinityPropagation -from ._hierarchical import (ward_tree, AgglomerativeClustering, linkage_tree, - FeatureAgglomeration) -from ._k_means import k_means, KMeans, MiniBatchKMeans +from ._agglomerative import (ward_tree, AgglomerativeClustering, + linkage_tree, FeatureAgglomeration) +from ._kmeans import k_means, KMeans, MiniBatchKMeans from ._dbscan import dbscan, DBSCAN from ._optics import (OPTICS, cluster_optics_dbscan, compute_optics_graph, cluster_optics_xi) diff --git a/sklearn/cluster/_affinity_propagation.py b/sklearn/cluster/_affinity_propagation.py index 3393e0686bd02..9516c8e4bdd05 100644 --- a/sklearn/cluster/_affinity_propagation.py +++ b/sklearn/cluster/_affinity_propagation.py @@ -11,7 +11,7 @@ from ..exceptions import ConvergenceWarning from ..base import BaseEstimator, ClusterMixin from ..utils import as_float_array, check_array -from ..utils.validation import check_is_fitted +from ..utils.validation import check_is_fitted, _deprecate_positional_args from ..metrics import euclidean_distances from ..metrics import pairwise_distances_argmin @@ -242,51 +242,51 @@ class AffinityPropagation(ClusterMixin, BaseEstimator): Parameters ---------- - damping : float, optional, default: 0.5 + damping : float, default=0.5 Damping factor (between 0.5 and 1) is the extent to which the current value is maintained relative to incoming values (weighted 1 - damping). This in order to avoid numerical oscillations when updating these values (messages). - max_iter : int, optional, default: 200 + max_iter : int, default=200 Maximum number of iterations. - convergence_iter : int, optional, default: 15 + convergence_iter : int, default=15 Number of iterations with no change in the number of estimated clusters that stops the convergence. - copy : boolean, optional, default: True + copy : bool, default=True Make a copy of input data. - preference : array-like, shape (n_samples,) or float, optional + preference : array-like of shape (n_samples,) or float, default=None Preferences for each point - points with larger values of preferences are more likely to be chosen as exemplars. The number of exemplars, ie of clusters, is influenced by the input preferences value. If the preferences are not passed as arguments, they will be set to the median of the input similarities. - affinity : string, optional, default=``euclidean`` - Which affinity to use. At the moment ``precomputed`` and - ``euclidean`` are supported. ``euclidean`` uses the + affinity : {'euclidean', 'precomputed'}, default='euclidean' + Which affinity to use. At the moment 'precomputed' and + ``euclidean`` are supported. 'euclidean' uses the negative squared euclidean distance between points. - verbose : boolean, optional, default: False + verbose : bool, default=False Whether to be verbose. Attributes ---------- - cluster_centers_indices_ : array, shape (n_clusters,) + cluster_centers_indices_ : ndarray of shape (n_clusters,) Indices of cluster centers - cluster_centers_ : array, shape (n_clusters, n_features) + cluster_centers_ : ndarray of shape (n_clusters, n_features) Cluster centers (if affinity != ``precomputed``). - labels_ : array, shape (n_samples,) + labels_ : ndarray of shape (n_samples,) Labels of each point - affinity_matrix_ : array, shape (n_samples, n_samples) + affinity_matrix_ : ndarray of shape (n_samples, n_samples) Stores the affinity matrix used in ``fit``. n_iter_ : int @@ -334,8 +334,8 @@ class AffinityPropagation(ClusterMixin, BaseEstimator): Brendan J. Frey and Delbert Dueck, "Clustering by Passing Messages Between Data Points", Science Feb. 2007 """ - - def __init__(self, damping=.5, max_iter=200, convergence_iter=15, + @_deprecate_positional_args + def __init__(self, *, damping=.5, max_iter=200, convergence_iter=15, copy=True, preference=None, affinity='euclidean', verbose=False): @@ -374,7 +374,7 @@ def fit(self, X, y=None): accept_sparse = False else: accept_sparse = 'csr' - X = check_array(X, accept_sparse=accept_sparse) + X = self._validate_data(X, accept_sparse=accept_sparse) if self.affinity == "precomputed": self.affinity_matrix_ = X elif self.affinity == "euclidean": diff --git a/sklearn/cluster/_hierarchical.py b/sklearn/cluster/_agglomerative.py similarity index 89% rename from sklearn/cluster/_hierarchical.py rename to sklearn/cluster/_agglomerative.py index eb3b989c7c815..182ae4b481116 100644 --- a/sklearn/cluster/_hierarchical.py +++ b/sklearn/cluster/_agglomerative.py @@ -17,13 +17,15 @@ from ..base import BaseEstimator, ClusterMixin from ..metrics.pairwise import paired_distances, pairwise_distances from ..utils import check_array -from ..utils.validation import check_memory +from ..utils.validation import check_memory, _deprecate_positional_args +from ..neighbors import DistanceMetric +from ..neighbors._dist_metrics import METRIC_MAPPING from . import _hierarchical_fast as _hierarchical from ._feature_agglomeration import AgglomerationTransform from ..utils._fast_dict import IntFloatDict from ..utils.fixes import _astype_copy_false -from ..utils import deprecated + ############################################################################### # For non fully-connected graphs @@ -107,7 +109,7 @@ def _single_linkage_tree(connectivity, n_samples, n_nodes, n_clusters, mst_array = np.vstack([mst.row, mst.col, mst.data]).T # Sort edges of the min_spanning_tree by weight - mst_array = mst_array[np.argsort(mst_array.T[2]), :] + mst_array = mst_array[np.argsort(mst_array.T[2], kind='mergesort'), :] # Convert edge list into standard hierarchical clustering format single_linkage_tree = _hierarchical._single_linkage_label(mst_array) @@ -247,8 +249,8 @@ def ward_tree(X, connectivity=None, n_clusters=None, return_distance=False): else: if n_clusters > n_samples: raise ValueError('Cannot provide more clusters than samples. ' - '%i n_clusters was asked, and there are %i samples.' - % (n_clusters, n_samples)) + '%i n_clusters was asked, and there are %i ' + 'samples.' % (n_clusters, n_samples)) n_nodes = 2 * n_samples - n_clusters # create inertia matrix @@ -451,8 +453,12 @@ def linkage_tree(X, connectivity=None, n_clusters=None, linkage='complete', if affinity == 'precomputed': # for the linkage function of hierarchy to work on precomputed # data, provide as first argument an ndarray of the shape returned - # by pdist: it is a flat array containing the upper triangular of - # the distance matrix. + # by sklearn.metrics.pairwise_distances. + if X.shape[0] != X.shape[1]: + raise ValueError( + 'Distance matrix should be square, ' + 'Got matrix of shape {X.shape}' + ) i, j = np.triu_indices(X.shape[0], k=1) X = X[i, j] elif affinity == 'l2': @@ -464,7 +470,25 @@ def linkage_tree(X, connectivity=None, n_clusters=None, linkage='complete', X = affinity(X) i, j = np.triu_indices(X.shape[0], k=1) X = X[i, j] - out = hierarchy.linkage(X, method=linkage, metric=affinity) + if (linkage == 'single' + and affinity != 'precomputed' + and not callable(affinity) + and affinity in METRIC_MAPPING): + + # We need the fast cythonized metric from neighbors + dist_metric = DistanceMetric.get_metric(affinity) + + # The Cython routines used require contiguous arrays + X = np.ascontiguousarray(X, dtype=np.double) + + mst = _hierarchical.mst_linkage_core(X, dist_metric) + # Sort edges of the min_spanning_tree by weight + mst = mst[np.argsort(mst.T[2], kind='mergesort'), :] + + # Convert edge list into standard hierarchical clustering format + out = _hierarchical.single_linkage_label(mst) + else: + out = hierarchy.linkage(X, method=linkage, metric=affinity) children_ = out[:, :2].astype(np.int, copy=False) if return_distance: @@ -663,23 +687,23 @@ class AgglomerativeClustering(ClusterMixin, BaseEstimator): Parameters ---------- - n_clusters : int or None, optional (default=2) + n_clusters : int or None, default=2 The number of clusters to find. It must be ``None`` if ``distance_threshold`` is not ``None``. - affinity : string or callable, default: "euclidean" + affinity : str or callable, default='euclidean' Metric used to compute the linkage. Can be "euclidean", "l1", "l2", "manhattan", "cosine", or "precomputed". If linkage is "ward", only "euclidean" is accepted. If "precomputed", a distance matrix (instead of a similarity matrix) is needed as input for the fit method. - memory : None, str or object with the joblib.Memory interface, optional + memory : str or object with the joblib.Memory interface, default=None Used to cache the output of the computation of the tree. By default, no caching is done. If a string is given, it is the path to the caching directory. - connectivity : array-like or callable, optional + connectivity : array-like or callable, default=None Connectivity matrix. Defines for each sample the neighboring samples following a given structure of the data. This can be a connectivity matrix itself or a callable that transforms @@ -687,17 +711,19 @@ class AgglomerativeClustering(ClusterMixin, BaseEstimator): kneighbors_graph. Default is None, i.e, the hierarchical clustering algorithm is unstructured. - compute_full_tree : bool or 'auto' (optional) - Stop early the construction of the tree at n_clusters. This is - useful to decrease computation time if the number of clusters is - not small compared to the number of samples. This option is - useful only when specifying a connectivity matrix. Note also that - when varying the number of clusters and using caching, it may - be advantageous to compute the full tree. It must be ``True`` if - ``distance_threshold`` is not ``None``. - - linkage : {"ward", "complete", "average", "single"}, optional \ - (default="ward") + compute_full_tree : 'auto' or bool, default='auto' + Stop early the construction of the tree at n_clusters. This is useful + to decrease computation time if the number of clusters is not small + compared to the number of samples. This option is useful only when + specifying a connectivity matrix. Note also that when varying the + number of clusters and using caching, it may be advantageous to compute + the full tree. It must be ``True`` if ``distance_threshold`` is not + ``None``. By default `compute_full_tree` is "auto", which is equivalent + to `True` when `distance_threshold` is not `None` or that `n_clusters` + is inferior to the maximum between 100 or `0.02 * n_samples`. + Otherwise, "auto" is equivalent to `False`. + + linkage : {"ward", "complete", "average", "single"}, default="ward" Which linkage criterion to use. The linkage criterion determines which distance to use between sets of observation. The algorithm will merge the pairs of cluster that minimize this criterion. @@ -710,7 +736,7 @@ class AgglomerativeClustering(ClusterMixin, BaseEstimator): - single uses the minimum of the distances between all observations of the two sets. - distance_threshold : float, optional (default=None) + distance_threshold : float, default=None The linkage distance threshold above which, clusters will not be merged. If not ``None``, ``n_clusters`` must be ``None`` and ``compute_full_tree`` must be ``True``. @@ -724,7 +750,7 @@ class AgglomerativeClustering(ClusterMixin, BaseEstimator): ``distance_threshold=None``, it will be equal to the given ``n_clusters``. - labels_ : array [n_samples] + labels_ : ndarray of shape (n_samples) cluster labels for each point n_leaves_ : int @@ -733,7 +759,7 @@ class AgglomerativeClustering(ClusterMixin, BaseEstimator): n_connected_components_ : int The estimated number of connected components in the graph. - children_ : array-like, shape (n_samples-1, 2) + children_ : array-like of shape (n_samples-1, 2) The children of each non-leaf node. Values less than `n_samples` correspond to leaves of the tree which are the original samples. A node `i` greater than or equal to `n_samples` is a non-leaf @@ -754,8 +780,8 @@ class AgglomerativeClustering(ClusterMixin, BaseEstimator): array([1, 1, 1, 0, 0, 0]) """ - - def __init__(self, n_clusters=2, affinity="euclidean", + @_deprecate_positional_args + def __init__(self, n_clusters=2, *, affinity="euclidean", memory=None, connectivity=None, compute_full_tree='auto', linkage='ward', distance_threshold=None): @@ -767,13 +793,6 @@ def __init__(self, n_clusters=2, affinity="euclidean", self.linkage = linkage self.affinity = affinity - @deprecated("The ``n_components_`` attribute was deprecated " - "in favor of ``n_connected_components_`` in 0.21 " - "and will be removed in 0.23.") - @property - def n_components_(self): - return self.n_connected_components_ - def fit(self, X, y=None): """Fit the hierarchical clustering from features, or distance matrix. @@ -790,7 +809,7 @@ def fit(self, X, y=None): ------- self """ - X = check_array(X, ensure_min_samples=2, estimator=self) + X = self._validate_data(X, ensure_min_samples=2, estimator=self) memory = check_memory(self.memory) if self.n_clusters is not None and self.n_clusters <= 0: @@ -909,21 +928,21 @@ class FeatureAgglomeration(AgglomerativeClustering, AgglomerationTransform): Parameters ---------- - n_clusters : int or None, optional (default=2) + n_clusters : int, default=2 The number of clusters to find. It must be ``None`` if ``distance_threshold`` is not ``None``. - affinity : string or callable, default "euclidean" + affinity : str or callable, default='euclidean' Metric used to compute the linkage. Can be "euclidean", "l1", "l2", "manhattan", "cosine", or 'precomputed'. If linkage is "ward", only "euclidean" is accepted. - memory : None, str or object with the joblib.Memory interface, optional + memory : str or object with the joblib.Memory interface, default=None Used to cache the output of the computation of the tree. By default, no caching is done. If a string is given, it is the path to the caching directory. - connectivity : array-like or callable, optional + connectivity : array-like or callable, default=None Connectivity matrix. Defines for each feature the neighboring features following a given structure of the data. This can be a connectivity matrix itself or a callable that transforms @@ -931,17 +950,19 @@ class FeatureAgglomeration(AgglomerativeClustering, AgglomerationTransform): kneighbors_graph. Default is None, i.e, the hierarchical clustering algorithm is unstructured. - compute_full_tree : bool or 'auto', optional, default "auto" - Stop early the construction of the tree at n_clusters. This is - useful to decrease computation time if the number of clusters is - not small compared to the number of features. This option is - useful only when specifying a connectivity matrix. Note also that - when varying the number of clusters and using caching, it may - be advantageous to compute the full tree. It must be ``True`` if - ``distance_threshold`` is not ``None``. - - linkage : {"ward", "complete", "average", "single"}, optional\ - (default="ward") + compute_full_tree : 'auto' or bool, optional, default='auto' + Stop early the construction of the tree at n_clusters. This is useful + to decrease computation time if the number of clusters is not small + compared to the number of features. This option is useful only when + specifying a connectivity matrix. Note also that when varying the + number of clusters and using caching, it may be advantageous to compute + the full tree. It must be ``True`` if ``distance_threshold`` is not + ``None``. By default `compute_full_tree` is "auto", which is equivalent + to `True` when `distance_threshold` is not `None` or that `n_clusters` + is inferior to the maximum between 100 or `0.02 * n_samples`. + Otherwise, "auto" is equivalent to `False`. + + linkage : {'ward', 'complete', 'average', 'single'}, default='ward' Which linkage criterion to use. The linkage criterion determines which distance to use between sets of features. The algorithm will merge the pairs of cluster that minimize this criterion. @@ -954,12 +975,12 @@ class FeatureAgglomeration(AgglomerativeClustering, AgglomerationTransform): - single uses the minimum of the distances between all observations of the two sets. - pooling_func : callable, default np.mean + pooling_func : callable, default=np.mean This combines the values of agglomerated features into a single value, and should accept an array of shape [M, N] and the keyword argument `axis=1`, and reduce it to an array of size [M]. - distance_threshold : float, optional (default=None) + distance_threshold : float, default=None The linkage distance threshold above which, clusters will not be merged. If not ``None``, ``n_clusters`` must be ``None`` and ``compute_full_tree`` must be ``True``. @@ -973,7 +994,7 @@ class FeatureAgglomeration(AgglomerativeClustering, AgglomerationTransform): ``distance_threshold=None``, it will be equal to the given ``n_clusters``. - labels_ : array-like, (n_features,) + labels_ : array-like of (n_features,) cluster labels for each feature. n_leaves_ : int @@ -982,7 +1003,7 @@ class FeatureAgglomeration(AgglomerativeClustering, AgglomerationTransform): n_connected_components_ : int The estimated number of connected components in the graph. - children_ : array-like, shape (n_nodes-1, 2) + children_ : array-like of shape (n_nodes-1, 2) The children of each non-leaf node. Values less than `n_features` correspond to leaves of the tree which are the original samples. A node `i` greater than or equal to `n_features` is a non-leaf @@ -990,7 +1011,7 @@ class FeatureAgglomeration(AgglomerativeClustering, AgglomerationTransform): at the i-th iteration, children[i][0] and children[i][1] are merged to form node `n_features + i` - distances_ : array-like, shape (n_nodes-1,) + distances_ : array-like of shape (n_nodes-1,) Distances between nodes in the corresponding place in `children_`. Only computed if distance_threshold is not None. @@ -1008,8 +1029,8 @@ class FeatureAgglomeration(AgglomerativeClustering, AgglomerationTransform): >>> X_reduced.shape (1797, 32) """ - - def __init__(self, n_clusters=2, affinity="euclidean", + @_deprecate_positional_args + def __init__(self, n_clusters=2, *, affinity="euclidean", memory=None, connectivity=None, compute_full_tree='auto', linkage='ward', pooling_func=np.mean, @@ -1034,9 +1055,14 @@ def fit(self, X, y=None, **params): ------- self """ - X = check_array(X, accept_sparse=['csr', 'csc', 'coo'], - ensure_min_features=2, estimator=self) - return AgglomerativeClustering.fit(self, X.T, **params) + X = self._validate_data(X, accept_sparse=['csr', 'csc', 'coo'], + ensure_min_features=2, estimator=self) + # save n_features_in_ attribute here to reset it after, because it will + # be overridden in AgglomerativeClustering since we passed it X.T. + n_features_in_ = self.n_features_in_ + AgglomerativeClustering.fit(self, X.T, **params) + self.n_features_in_ = n_features_in_ + return self @property def fit_predict(self): diff --git a/sklearn/cluster/_bicluster.py b/sklearn/cluster/_bicluster.py index 5bfd335549012..c98272d6aae33 100644 --- a/sklearn/cluster/_bicluster.py +++ b/sklearn/cluster/_bicluster.py @@ -3,6 +3,7 @@ # License: BSD 3 clause from abc import ABCMeta, abstractmethod +import warnings import numpy as np @@ -17,7 +18,8 @@ from ..utils.extmath import (make_nonnegative, randomized_svd, safe_sparse_dot) -from ..utils.validation import assert_all_finite, check_array +from ..utils.validation import (assert_all_finite, check_array, + _deprecate_positional_args) __all__ = ['SpectralCoclustering', @@ -88,7 +90,7 @@ class BaseSpectral(BiclusterMixin, BaseEstimator, metaclass=ABCMeta): @abstractmethod def __init__(self, n_clusters=3, svd_method="randomized", n_svd_vecs=None, mini_batch=False, init="k-means++", - n_init=10, n_jobs=None, random_state=None): + n_init=10, n_jobs='deprecated', random_state=None): self.n_clusters = n_clusters self.svd_method = svd_method self.n_svd_vecs = n_svd_vecs @@ -115,7 +117,11 @@ def fit(self, X, y=None): y : Ignored """ - X = check_array(X, accept_sparse='csr', dtype=np.float64) + if self.n_jobs != 'deprecated': + warnings.warn("'n_jobs' was deprecated in version 0.23 and will be" + " removed in 0.25.", FutureWarning) + + X = self._validate_data(X, accept_sparse='csr', dtype=np.float64) self._check_parameters() self._fit(X) return self @@ -191,10 +197,10 @@ class SpectralCoclustering(BaseSpectral): Parameters ---------- - n_clusters : integer, optional, default: 3 + n_clusters : int, default=3 The number of biclusters to find. - svd_method : string, optional, default: 'randomized' + svd_method : {'randomized', 'arpack'}, default='randomized' Selects the algorithm for finding singular vectors. May be 'randomized' or 'arpack'. If 'randomized', use :func:`sklearn.utils.extmath.randomized_svd`, which may be faster @@ -202,20 +208,21 @@ class SpectralCoclustering(BaseSpectral): :func:`scipy.sparse.linalg.svds`, which is more accurate, but possibly slower in some cases. - n_svd_vecs : int, optional, default: None + n_svd_vecs : int, default=None Number of vectors to use in calculating the SVD. Corresponds to `ncv` when `svd_method=arpack` and `n_oversamples` when `svd_method` is 'randomized`. - mini_batch : bool, optional, default: False + mini_batch : bool, default=False Whether to use mini-batch k-means, which is faster but may get different results. - init : {'k-means++', 'random' or an ndarray} - Method for initialization of k-means algorithm; defaults to - 'k-means++'. + init : {'k-means++', 'random', or ndarray of shape \ + (n_clusters, n_features), default='k-means++' + Method for initialization of k-means algorithm; defaults to + 'k-means++'. - n_init : int, optional, default: 10 + n_init : int, default=10 Number of random initializations that are tried with the k-means algorithm. @@ -223,7 +230,7 @@ class SpectralCoclustering(BaseSpectral): chosen and the algorithm runs once. Otherwise, the algorithm is run for each initialization and the best solution chosen. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of jobs to use for the computation. This works by breaking down the pairwise matrix into n_jobs even slices and computing them in parallel. @@ -232,24 +239,28 @@ class SpectralCoclustering(BaseSpectral): ``-1`` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance or None (default) + .. deprecated:: 0.23 + ``n_jobs`` was deprecated in version 0.23 and will be removed in + 0.25. + + random_state : int, RandomState instance, default=None Used for randomizing the singular value decomposition and the k-means initialization. Use an int to make the randomness deterministic. See :term:`Glossary `. Attributes ---------- - rows_ : array-like, shape (n_row_clusters, n_rows) + rows_ : array-like of shape (n_row_clusters, n_rows) Results of the clustering. `rows[i, r]` is True if cluster `i` contains row `r`. Available only after calling ``fit``. - columns_ : array-like, shape (n_column_clusters, n_columns) + columns_ : array-like of shape (n_column_clusters, n_columns) Results of the clustering, like `rows`. - row_labels_ : array-like, shape (n_rows,) + row_labels_ : array-like of shape (n_rows,) The bicluster label of each row. - column_labels_ : array-like, shape (n_cols,) + column_labels_ : array-like of shape (n_cols,) The bicluster label of each column. Examples @@ -259,9 +270,9 @@ class SpectralCoclustering(BaseSpectral): >>> X = np.array([[1, 1], [2, 1], [1, 0], ... [4, 7], [3, 5], [3, 6]]) >>> clustering = SpectralCoclustering(n_clusters=2, random_state=0).fit(X) - >>> clustering.row_labels_ + >>> clustering.row_labels_ #doctest: +SKIP array([0, 1, 1, 0, 0, 0], dtype=int32) - >>> clustering.column_labels_ + >>> clustering.column_labels_ #doctest: +SKIP array([0, 0], dtype=int32) >>> clustering SpectralCoclustering(n_clusters=2, random_state=0) @@ -274,9 +285,10 @@ class SpectralCoclustering(BaseSpectral): `__. """ - def __init__(self, n_clusters=3, svd_method='randomized', + @_deprecate_positional_args + def __init__(self, n_clusters=3, *, svd_method='randomized', n_svd_vecs=None, mini_batch=False, init='k-means++', - n_init=10, n_jobs=None, random_state=None): + n_init=10, n_jobs='deprecated', random_state=None): super().__init__(n_clusters, svd_method, n_svd_vecs, @@ -319,26 +331,28 @@ class SpectralBiclustering(BaseSpectral): Parameters ---------- - n_clusters : integer or tuple (n_row_clusters, n_column_clusters) + n_clusters : int or tuple (n_row_clusters, n_column_clusters), default=3 The number of row and column clusters in the checkerboard structure. - method : string, optional, default: 'bistochastic' + method : {'bistochastic', 'scale', 'log'}, default='bistochastic' Method of normalizing and converting singular vectors into biclusters. May be one of 'scale', 'bistochastic', or 'log'. The authors recommend using 'log'. If the data is sparse, however, log normalization will not work, which is why the - default is 'bistochastic'. CAUTION: if `method='log'`, the - data must not be sparse. + default is 'bistochastic'. + + .. warning:: + if `method='log'`, the data must be sparse. - n_components : integer, optional, default: 6 + n_components : int, default=6 Number of singular vectors to check. - n_best : integer, optional, default: 3 + n_best : int, default=3 Number of best singular vectors to which to project the data for clustering. - svd_method : string, optional, default: 'randomized' + svd_method : {'randomized', 'arpack'}, default='randomized' Selects the algorithm for finding singular vectors. May be 'randomized' or 'arpack'. If 'randomized', uses :func:`~sklearn.utils.extmath.randomized_svd`, which may be faster @@ -346,20 +360,21 @@ class SpectralBiclustering(BaseSpectral): `scipy.sparse.linalg.svds`, which is more accurate, but possibly slower in some cases. - n_svd_vecs : int, optional, default: None + n_svd_vecs : int, default=None Number of vectors to use in calculating the SVD. Corresponds to `ncv` when `svd_method=arpack` and `n_oversamples` when `svd_method` is 'randomized`. - mini_batch : bool, optional, default: False + mini_batch : bool, default=False Whether to use mini-batch k-means, which is faster but may get different results. - init : {'k-means++', 'random' or an ndarray} - Method for initialization of k-means algorithm; defaults to - 'k-means++'. + init : {'k-means++', 'random'} or ndarray of (n_clusters, n_features), \ + default='k-means++' + Method for initialization of k-means algorithm; defaults to + 'k-means++'. - n_init : int, optional, default: 10 + n_init : int, default=10 Number of random initializations that are tried with the k-means algorithm. @@ -367,7 +382,7 @@ class SpectralBiclustering(BaseSpectral): chosen and the algorithm runs once. Otherwise, the algorithm is run for each initialization and the best solution chosen. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of jobs to use for the computation. This works by breaking down the pairwise matrix into n_jobs even slices and computing them in parallel. @@ -376,24 +391,28 @@ class SpectralBiclustering(BaseSpectral): ``-1`` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance or None (default) + .. deprecated:: 0.23 + ``n_jobs`` was deprecated in version 0.23 and will be removed in + 0.25. + + random_state : int, RandomState instance, default=None Used for randomizing the singular value decomposition and the k-means initialization. Use an int to make the randomness deterministic. See :term:`Glossary `. Attributes ---------- - rows_ : array-like, shape (n_row_clusters, n_rows) + rows_ : array-like of shape (n_row_clusters, n_rows) Results of the clustering. `rows[i, r]` is True if cluster `i` contains row `r`. Available only after calling ``fit``. - columns_ : array-like, shape (n_column_clusters, n_columns) + columns_ : array-like of shape (n_column_clusters, n_columns) Results of the clustering, like `rows`. - row_labels_ : array-like, shape (n_rows,) + row_labels_ : array-like of shape (n_rows,) Row partition labels. - column_labels_ : array-like, shape (n_cols,) + column_labels_ : array-like of shape (n_cols,) Column partition labels. Examples @@ -418,10 +437,11 @@ class SpectralBiclustering(BaseSpectral): `__. """ - def __init__(self, n_clusters=3, method='bistochastic', + @_deprecate_positional_args + def __init__(self, n_clusters=3, *, method='bistochastic', n_components=6, n_best=3, svd_method='randomized', n_svd_vecs=None, mini_batch=False, init='k-means++', - n_init=10, n_jobs=None, random_state=None): + n_init=10, n_jobs='deprecated', random_state=None): super().__init__(n_clusters, svd_method, n_svd_vecs, diff --git a/sklearn/cluster/_birch.py b/sklearn/cluster/_birch.py index 6ac9ec334a734..1d81dafc7504d 100644 --- a/sklearn/cluster/_birch.py +++ b/sklearn/cluster/_birch.py @@ -4,15 +4,17 @@ # License: BSD 3 clause import warnings +import numbers import numpy as np from scipy import sparse from math import sqrt +from ..metrics import pairwise_distances_argmin from ..metrics.pairwise import euclidean_distances from ..base import TransformerMixin, ClusterMixin, BaseEstimator from ..utils import check_array -from ..utils.extmath import row_norms, safe_sparse_dot -from ..utils.validation import check_is_fitted +from ..utils.extmath import row_norms +from ..utils.validation import check_is_fitted, _deprecate_positional_args from ..exceptions import ConvergenceWarning from . import AgglomerativeClustering @@ -48,10 +50,12 @@ def _split_node(node, threshold, branching_factor): new_subcluster1 = _CFSubcluster() new_subcluster2 = _CFSubcluster() new_node1 = _CFNode( - threshold, branching_factor, is_leaf=node.is_leaf, + threshold=threshold, branching_factor=branching_factor, + is_leaf=node.is_leaf, n_features=node.n_features) new_node2 = _CFNode( - threshold, branching_factor, is_leaf=node.is_leaf, + threshold=threshold, branching_factor=branching_factor, + is_leaf=node.is_leaf, n_features=node.n_features) new_subcluster1.child_ = new_node1 new_subcluster2.child_ = new_node2 @@ -132,7 +136,7 @@ class _CFNode: view of ``init_sq_norm_``. """ - def __init__(self, threshold, branching_factor, is_leaf, n_features): + def __init__(self, *, threshold, branching_factor, is_leaf, n_features): self.threshold = threshold self.branching_factor = branching_factor self.is_leaf = is_leaf @@ -273,7 +277,7 @@ class _CFSubcluster: Squared norm of the subcluster. Used to prevent recomputing when pairwise minimum distances are computed. """ - def __init__(self, linear_sum=None): + def __init__(self, *, linear_sum=None): if linear_sum is None: self.n_samples_ = 0 self.squared_sum_ = 0.0 @@ -334,20 +338,20 @@ class Birch(ClusterMixin, TransformerMixin, BaseEstimator): Parameters ---------- - threshold : float, default 0.5 + threshold : float, default=0.5 The radius of the subcluster obtained by merging a new sample and the closest subcluster should be lesser than the threshold. Otherwise a new subcluster is started. Setting this value to be very low promotes splitting and vice-versa. - branching_factor : int, default 50 + branching_factor : int, default=50 Maximum number of CF subclusters in each node. If a new samples enters such that the number of subclusters exceed the branching_factor then that node is split into two nodes with the subclusters redistributed in each. The parent subcluster of that node is removed and two new subclusters are added as parents of the 2 split nodes. - n_clusters : int, instance of sklearn.cluster model, default 3 + n_clusters : int, instance of sklearn.cluster model, default=3 Number of clusters after the final clustering step, which treats the subclusters from the leaves as new samples. @@ -361,10 +365,10 @@ class Birch(ClusterMixin, TransformerMixin, BaseEstimator): - `int` : the model fit is :class:`AgglomerativeClustering` with `n_clusters` set to be equal to the int. - compute_labels : bool, default True + compute_labels : bool, default=True Whether or not to compute labels for each fit. - copy : bool, default True + copy : bool, default=True Whether or not to make a copy of the given data. If set to False, the initial data will be overwritten. @@ -388,25 +392,12 @@ class Birch(ClusterMixin, TransformerMixin, BaseEstimator): if partial_fit is used instead of fit, they are assigned to the last batch of data. - Examples + See Also -------- - >>> from sklearn.cluster import Birch - >>> X = [[0, 1], [0.3, 1], [-0.3, 1], [0, -1], [0.3, -1], [-0.3, -1]] - >>> brc = Birch(n_clusters=None) - >>> brc.fit(X) - Birch(n_clusters=None) - >>> brc.predict(X) - array([0, 0, 0, 1, 1, 1]) - References - ---------- - * Tian Zhang, Raghu Ramakrishnan, Maron Livny - BIRCH: An efficient data clustering method for large databases. - https://www.cs.sfu.ca/CourseCentral/459/han/papers/zhang96.pdf - - * Roberto Perdisci - JBirch - Java implementation of BIRCH clustering algorithm - https://code.google.com/archive/p/jbirch + MiniBatchKMeans + Alternative implementation that does incremental updates + of the centers' positions using mini-batches. Notes ----- @@ -421,9 +412,29 @@ class Birch(ClusterMixin, TransformerMixin, BaseEstimator): to it and the linear sum, squared sum and the number of samples of that subcluster are updated. This is done recursively till the properties of the leaf node are updated. - """ - def __init__(self, threshold=0.5, branching_factor=50, n_clusters=3, + References + ---------- + * Tian Zhang, Raghu Ramakrishnan, Maron Livny + BIRCH: An efficient data clustering method for large databases. + https://www.cs.sfu.ca/CourseCentral/459/han/papers/zhang96.pdf + + * Roberto Perdisci + JBirch - Java implementation of BIRCH clustering algorithm + https://code.google.com/archive/p/jbirch + + Examples + -------- + >>> from sklearn.cluster import Birch + >>> X = [[0, 1], [0.3, 1], [-0.3, 1], [0, -1], [0.3, -1], [-0.3, -1]] + >>> brc = Birch(n_clusters=None) + >>> brc.fit(X) + Birch(n_clusters=None) + >>> brc.predict(X) + array([0, 0, 0, 1, 1, 1]) + """ + @_deprecate_positional_args + def __init__(self, *, threshold=0.5, branching_factor=50, n_clusters=3, compute_labels=True, copy=True): self.threshold = threshold self.branching_factor = branching_factor @@ -441,13 +452,18 @@ def fit(self, X, y=None): Input data. y : Ignored + Not used, present here for API consistency by convention. + Returns + ------- + self + Fitted estimator. """ self.fit_, self.partial_fit_ = True, False return self._fit(X) def _fit(self, X): - X = check_array(X, accept_sparse='csr', copy=self.copy) + X = self._validate_data(X, accept_sparse='csr', copy=self.copy) threshold = self.threshold branching_factor = self.branching_factor @@ -461,11 +477,14 @@ def _fit(self, X): has_root = getattr(self, 'root_', None) if getattr(self, 'fit_') or (partial_fit and not has_root): # The first root is the leaf. Manipulate this object throughout. - self.root_ = _CFNode(threshold, branching_factor, is_leaf=True, + self.root_ = _CFNode(threshold=threshold, + branching_factor=branching_factor, + is_leaf=True, n_features=n_features) # To enable getting back subclusters. - self.dummy_leaf_ = _CFNode(threshold, branching_factor, + self.dummy_leaf_ = _CFNode(threshold=threshold, + branching_factor=branching_factor, is_leaf=True, n_features=n_features) self.dummy_leaf_.next_leaf_ = self.root_ self.root_.prev_leaf_ = self.dummy_leaf_ @@ -484,7 +503,8 @@ def _fit(self, X): new_subcluster1, new_subcluster2 = _split_node( self.root_, threshold, branching_factor) del self.root_ - self.root_ = _CFNode(threshold, branching_factor, + self.root_ = _CFNode(threshold=threshold, + branching_factor=branching_factor, is_leaf=False, n_features=n_features) self.root_.append_subcluster(new_subcluster1) @@ -524,7 +544,12 @@ def partial_fit(self, X=None, y=None): step is done. y : Ignored + Not used, present here for API consistency by convention. + Returns + ------- + self + Fitted estimator. """ self.partial_fit_, self.fit_ = True, False if X is None: @@ -562,10 +587,12 @@ def predict(self, X): """ X = check_array(X, accept_sparse='csr') self._check_fit(X) - reduced_distance = safe_sparse_dot(X, self.subcluster_centers_.T) - reduced_distance *= -2 - reduced_distance += self._subcluster_norms - return self.subcluster_labels_[np.argmin(reduced_distance, axis=1)] + kwargs = {'Y_norm_squared': self._subcluster_norms} + return self.subcluster_labels_[ + pairwise_distances_argmin(X, + self.subcluster_centers_, + metric_kwargs=kwargs) + ] def transform(self, X): """ @@ -597,7 +624,7 @@ def _global_clustering(self, X=None): # Preprocessing for the global clustering. not_enough_centroids = False - if isinstance(clusterer, int): + if isinstance(clusterer, numbers.Integral): clusterer = AgglomerativeClustering( n_clusters=self.n_clusters) # There is no need to perform the global clustering step. diff --git a/sklearn/cluster/_dbscan.py b/sklearn/cluster/_dbscan.py index 3b3ccb1fbe6dc..52c962052f9bc 100644 --- a/sklearn/cluster/_dbscan.py +++ b/sklearn/cluster/_dbscan.py @@ -15,7 +15,7 @@ from ..base import BaseEstimator, ClusterMixin from ..utils import check_array -from ..utils.validation import _check_sample_weight +from ..utils.validation import _check_sample_weight, _deprecate_positional_args from ..neighbors import NearestNeighbors from ._dbscan_inner import dbscan_inner @@ -52,7 +52,8 @@ def dbscan(X, eps=0.5, min_samples=5, metric='minkowski', metric_params=None, the options allowed by :func:`sklearn.metrics.pairwise_distances` for its metric parameter. If metric is "precomputed", X is assumed to be a distance matrix and - must be square during fit. X may be a :term:`Glossary `, + must be square during fit. + X may be a :term:`sparse graph `, in which case only "nonzero" elements may be considered neighbors. metric_params : dict, optional @@ -82,10 +83,11 @@ def dbscan(X, eps=0.5, min_samples=5, metric='minkowski', metric_params=None, Note that weights are absolute, and default to 1. n_jobs : int or None, optional (default=None) - The number of parallel jobs to run for neighbors search. - ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. - ``-1`` means using all processors. See :term:`Glossary ` - for more details. + The number of parallel jobs to run for neighbors search. ``None`` means + 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means + using all processors. See :term:`Glossary ` for more details. + If precomputed distance are used, parallel execution is not available + and thus n_jobs will have no effect. Returns ------- @@ -156,18 +158,18 @@ class DBSCAN(ClusterMixin, BaseEstimator): Parameters ---------- - eps : float, optional + eps : float, default=0.5 The maximum distance between two samples for one to be considered as in the neighborhood of the other. This is not a maximum bound on the distances of points within a cluster. This is the most important DBSCAN parameter to choose appropriately for your data set and distance function. - min_samples : int, optional + min_samples : int, default=5 The number of samples (or total weight) in a neighborhood for a point to be considered as a core point. This includes the point itself. - metric : string, or callable + metric : string, or callable, default='euclidean' The metric to use when calculating distance between instances in a feature array. If metric is a string or callable, it must be one of the options allowed by :func:`sklearn.metrics.pairwise_distances` for @@ -179,27 +181,27 @@ class DBSCAN(ClusterMixin, BaseEstimator): .. versionadded:: 0.17 metric *precomputed* to accept precomputed sparse matrix. - metric_params : dict, optional + metric_params : dict, default=None Additional keyword arguments for the metric function. .. versionadded:: 0.19 - algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional + algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto' The algorithm to be used by the NearestNeighbors module to compute pointwise distances and find nearest neighbors. See NearestNeighbors module documentation for details. - leaf_size : int, optional (default = 30) + leaf_size : int, default=30 Leaf size passed to BallTree or cKDTree. This can affect the speed of the construction and query, as well as the memory required to store the tree. The optimal value depends on the nature of the problem. - p : float, optional + p : float, default=None The power of the Minkowski metric to be used to calculate distance between points. - n_jobs : int or None, optional (default=None) + n_jobs : int or None, default=None The number of parallel jobs to run. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` @@ -269,8 +271,8 @@ class DBSCAN(ClusterMixin, BaseEstimator): DBSCAN revisited, revisited: why and how you should (still) use DBSCAN. ACM Transactions on Database Systems (TODS), 42(3), 19. """ - - def __init__(self, eps=0.5, min_samples=5, metric='euclidean', + @_deprecate_positional_args + def __init__(self, eps=0.5, *, min_samples=5, metric='euclidean', metric_params=None, algorithm='auto', leaf_size=30, p=None, n_jobs=None): self.eps = eps @@ -307,7 +309,7 @@ def fit(self, X, y=None, sample_weight=None): self """ - X = check_array(X, accept_sparse='csr') + X = self._validate_data(X, accept_sparse='csr') if not self.eps > 0.0: raise ValueError("eps must be positive.") diff --git a/sklearn/cluster/_hierarchical_fast.pyx b/sklearn/cluster/_hierarchical_fast.pyx index 3dd02d5aaa5ae..ec8c96410c25c 100644 --- a/sklearn/cluster/_hierarchical_fast.pyx +++ b/sklearn/cluster/_hierarchical_fast.pyx @@ -13,6 +13,7 @@ ctypedef np.int8_t INT8 np.import_array() +from ..neighbors._dist_metrics cimport DistanceMetric from ..utils._fast_dict cimport IntFloatDict # C++ @@ -26,6 +27,8 @@ ctypedef np.float64_t DTYPE_t ITYPE = np.intp ctypedef np.intp_t ITYPE_t +from numpy.math cimport INFINITY + ############################################################################### # Utilities for computing the ward momentum @@ -446,3 +449,89 @@ def single_linkage_label(L): raise ValueError("Input MST array must be sorted by weight") return _single_linkage_label(L) + + +# Implements MST-LINKAGE-CORE from https://arxiv.org/abs/1109.2378 +@cython.boundscheck(False) +@cython.nonecheck(False) +def mst_linkage_core( + DTYPE_t [:, ::1] raw_data, + DistanceMetric dist_metric): + """ + Compute the necessary elements of a minimum spanning + tree for computation of single linkage clustering. This + represents the MST-LINKAGE-CORE algorithm (Figure 6) from + *Modern hierarchical, agglomerative clustering algorithms* + by Daniel Mullner (https://arxiv.org/abs/1109.2378). + + In contrast to the scipy implementation is never computes + a full distance matrix, generating distances only as they + are needed and releasing them when no longer needed. + + Parameters + ---------- + raw_data: array of shape (n_samples, n_features) + The array of feature data to be clustered. Must be C-aligned + + dist_metric: DistanceMetric + A DistanceMetric object conforming to the API from + ``sklearn.neighbors._dist_metrics.pxd`` that will be + used to compute distances. + + Returns + ------- + mst_core_data: array of shape (n_samples, 3) + An array providing information from which one + can either compute an MST, or the linkage hierarchy + very efficiently. See https://arxiv.org/abs/1109.2378 + algorithm MST-LINKAGE-CORE for more details. + """ + cdef: + ITYPE_t n_samples = raw_data.shape[0] + np.int8_t[:] in_tree = np.zeros(n_samples, dtype=np.int8) + DTYPE_t[:, ::1] result = np.zeros((n_samples - 1, 3)) + + np.ndarray label_filter + + ITYPE_t current_node = 0 + ITYPE_t new_node + ITYPE_t i + ITYPE_t j + ITYPE_t num_features = raw_data.shape[1] + + DTYPE_t right_value + DTYPE_t left_value + DTYPE_t new_distance + + DTYPE_t[:] current_distances = np.full(n_samples, INFINITY) + + for i in range(n_samples - 1): + + in_tree[current_node] = 1 + + new_distance = INFINITY + new_node = 0 + + for j in range(n_samples): + if in_tree[j]: + continue + + right_value = current_distances[j] + left_value = dist_metric.dist(&raw_data[current_node, 0], + &raw_data[j, 0], + num_features) + + if left_value < right_value: + current_distances[j] = left_value + + if current_distances[j] < new_distance: + new_distance = current_distances[j] + new_node = j + + result[i, 0] = current_node + result[i, 1] = new_node + result[i, 2] = new_distance + current_node = new_node + + return np.array(result) + diff --git a/sklearn/cluster/_k_means_elkan.pyx b/sklearn/cluster/_k_means_elkan.pyx index abf2ea8aeac8d..e95c8fe0490a4 100644 --- a/sklearn/cluster/_k_means_elkan.pyx +++ b/sklearn/cluster/_k_means_elkan.pyx @@ -1,38 +1,42 @@ -# cython: cdivision=True -# cython: boundscheck=False -# cython: wraparound=False +# cython: profile=True, boundscheck=False, wraparound=False, cdivision=True # # Author: Andreas Mueller # # Licence: BSD 3 clause +# TODO: We still need to use ndarrays instead of typed memoryviews when using +# fused types and when the array may be read-only (for instance when it's +# provided by the user). This is fixed in cython > 0.3. + import numpy as np cimport numpy as np cimport cython from cython cimport floating - +from cython.parallel import prange, parallel from libc.math cimport sqrt +from libc.stdlib cimport calloc, free +from libc.string cimport memset, memcpy -from ..metrics import euclidean_distances -from ._k_means_fast import _centers_dense +from ..utils.extmath import row_norms +from ._k_means_fast cimport _relocate_empty_clusters_dense +from ._k_means_fast cimport _relocate_empty_clusters_sparse +from ._k_means_fast cimport _euclidean_dense_dense +from ._k_means_fast cimport _euclidean_sparse_dense +from ._k_means_fast cimport _average_centers +from ._k_means_fast cimport _center_shift -cdef floating euclidean_dist(floating* a, floating* b, int n_features) nogil: - cdef floating result, tmp - result = 0 - cdef int i - for i in range(n_features): - tmp = (a[i] - b[i]) - result += tmp * tmp - return sqrt(result) +np.import_array() -cdef update_labels_distances_inplace( - floating* X, floating* centers, floating[:, :] center_half_distances, - int[:] labels, floating[:, :] lower_bounds, floating[:] upper_bounds, - Py_ssize_t n_samples, int n_features, int n_clusters): - """ - Calculate upper and lower bounds for each sample. +def _init_bounds_dense( + np.ndarray[floating, ndim=2, mode='c'] X, # IN + floating[:, ::1] centers, # IN + floating[:, ::1] center_half_distances, # IN + int[::1] labels, # OUT + floating[::1] upper_bounds, # OUT + floating[:, ::1] lower_bounds): # OUT + """Initialize upper and lower bounds for each sample for dense input data. Given X, centers and the pairwise distances divided by 2.0 between the centers this calculates the upper bounds and lower bounds for each sample. @@ -49,214 +53,586 @@ cdef update_labels_distances_inplace( Parameters ---------- - X : nd-array, shape (n_samples, n_features) + X : ndarray of shape (n_samples, n_features), dtype=floating The input data. - centers : nd-array, shape (n_clusters, n_features) + centers : ndarray of shape (n_clusters, n_features), dtype=floating The cluster centers. - center_half_distances : nd-array, shape (n_clusters, n_clusters) + center_half_distances : ndarray of shape (n_clusters, n_clusters), \ + dtype=floating The half of the distance between any 2 clusters centers. - labels : nd-array, shape(n_samples) + labels : ndarray of shape(n_samples), dtype=int The label for each sample. This array is modified in place. - lower_bounds : nd-array, shape(n_samples, n_clusters) - The lower bound on the distance between a sample and each cluster - center. It is modified in place. + upper_bounds : ndarray of shape(n_samples,), dtype=floating + The upper bound on the distance between each sample and its closest + cluster center. This array is modified in place. + + lower_bounds : ndarray, of shape(n_samples, n_clusters), dtype=floating + The lower bound on the distance between each sample and each cluster + center. This array is modified in place. + """ + cdef: + int n_samples = X.shape[0] + int n_clusters = centers.shape[0] + int n_features = X.shape[1] + + floating min_dist, dist + int best_cluster, i, j + + for i in range(n_samples): + best_cluster = 0 + min_dist = _euclidean_dense_dense(&X[i, 0], ¢ers[0, 0], + n_features, False) + lower_bounds[i, 0] = min_dist + for j in range(1, n_clusters): + if min_dist > center_half_distances[best_cluster, j]: + dist = _euclidean_dense_dense(&X[i, 0], ¢ers[j, 0], + n_features, False) + lower_bounds[i, j] = dist + if dist < min_dist: + min_dist = dist + best_cluster = j + labels[i] = best_cluster + upper_bounds[i] = min_dist + + +def _init_bounds_sparse( + X, # IN + floating[:, ::1] centers, # IN + floating[:, ::1] center_half_distances, # IN + int[::1] labels, # OUT + floating[::1] upper_bounds, # OUT + floating[:, ::1] lower_bounds): # OUT + """Initialize upper and lower bounds for each sample for sparse input data. + + Given X, centers and the pairwise distances divided by 2.0 between the + centers this calculates the upper bounds and lower bounds for each sample. + The upper bound for each sample is set to the distance between the sample + and the closest center. + + The lower bound for each sample is a one-dimensional array of n_clusters. + For each sample i assume that the previously assigned cluster is c1 and the + previous closest distance is dist, for a new cluster c2, the + lower_bound[i][c2] is set to distance between the sample and this new + cluster, if and only if dist > center_half_distances[c1][c2]. This prevents + computation of unnecessary distances for each sample to the clusters that + it is unlikely to be assigned to. + + Parameters + ---------- + X : sparse matrix of shape (n_samples, n_features), dtype=floating + The input data. Must be in CSR format. - upper_bounds : nd-array, shape(n_samples,) - The distance of each sample from its closest cluster center. This is - modified in place by the function. + centers : ndarray of shape (n_clusters, n_features), dtype=floating + The cluster centers. - n_samples : Py_ssize_t - The number of samples. + center_half_distances : ndarray of shape (n_clusters, n_clusters), \ + dtype=floating + The half of the distance between any 2 clusters centers. + + labels : ndarray of shape(n_samples), dtype=int + The label for each sample. This array is modified in place. - n_features : int - The number of features. + upper_bounds : ndarray of shape(n_samples,), dtype=floating + The upper bound on the distance between each sample and its closest + cluster center. This array is modified in place. - n_clusters : int - The number of clusters. + lower_bounds : ndarray of shape(n_samples, n_clusters), dtype=floating + The lower bound on the distance between each sample and each cluster + center. This array is modified in place. """ - # assigns closest center to X - # uses triangle inequality - cdef floating* x - cdef floating* c - cdef floating d_c, dist - cdef int c_x, j - cdef Py_ssize_t sample - for sample in range(n_samples): - # assign first cluster center - c_x = 0 - x = X + sample * n_features - d_c = euclidean_dist(x, centers, n_features) - lower_bounds[sample, 0] = d_c + cdef: + int n_samples = X.shape[0] + int n_clusters = centers.shape[0] + int n_features = X.shape[1] + + floating[::1] X_data = X.data + int[::1] X_indices = X.indices + int[::1] X_indptr = X.indptr + + floating min_dist, dist + int best_cluster, i, j + + floating[::1] centers_squared_norms = row_norms(centers, squared=True) + + for i in range(n_samples): + best_cluster = 0 + min_dist = _euclidean_sparse_dense( + X_data[X_indptr[i]: X_indptr[i + 1]], + X_indices[X_indptr[i]: X_indptr[i + 1]], + centers[0], centers_squared_norms[0], False) + + lower_bounds[i, 0] = min_dist for j in range(1, n_clusters): - if d_c > center_half_distances[c_x, j]: - c = centers + j * n_features - dist = euclidean_dist(x, c, n_features) - lower_bounds[sample, j] = dist - if dist < d_c: - d_c = dist - c_x = j - labels[sample] = c_x - upper_bounds[sample] = d_c - - -def k_means_elkan(np.ndarray[floating, ndim=2, mode='c'] X_, - np.ndarray[floating, ndim=1, mode='c'] sample_weight, - int n_clusters, - np.ndarray[floating, ndim=2, mode='c'] init, - float tol=1e-4, int max_iter=30, verbose=False): - """Run Elkan's k-means. + if min_dist > center_half_distances[best_cluster, j]: + dist = _euclidean_sparse_dense( + X_data[X_indptr[i]: X_indptr[i + 1]], + X_indices[X_indptr[i]: X_indptr[i + 1]], + centers[j], centers_squared_norms[j], False) + lower_bounds[i, j] = dist + if dist < min_dist: + min_dist = dist + best_cluster = j + labels[i] = best_cluster + upper_bounds[i] = min_dist + + +def _elkan_iter_chunked_dense( + np.ndarray[floating, ndim=2, mode='c'] X, # IN + floating[::1] sample_weight, # IN + floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # OUT + floating[::1] weight_in_clusters, # OUT + floating[:, ::1] center_half_distances, # IN + floating[::1] distance_next_center, # IN + floating[::1] upper_bounds, # INOUT + floating[:, ::1] lower_bounds, # INOUT + int[::1] labels, # INOUT + floating[::1] center_shift, # OUT + int n_threads, + bint update_centers=True): + """Single iteration of K-means Elkan algorithm with dense input. + + Update labels and centers (inplace), for one iteration, distributed + over data chunks. Parameters ---------- - X_ : nd-array, shape (n_samples, n_features) + X : ndarray of shape (n_samples, n_features), dtype=floating + The observations to cluster. - sample_weight : nd-array, shape (n_samples,) + sample_weight : ndarray of shape (n_samples,), dtype=floating The weights for each observation in X. - n_clusters : int - Number of clusters to find. + centers_old : ndarray of shape (n_clusters, n_features), dtype=floating + Centers before previous iteration, placeholder for the centers after + previous iteration. + + centers_new : ndarray of shape (n_clusters, n_features), dtype=floating + Centers after previous iteration, placeholder for the new centers + computed during this iteration. + + weight_in_clusters : ndarray of shape (n_clusters,), dtype=floating + Placeholder for the sums of the weights of every observation assigned + to each center. + + center_half_distances : ndarray of shape (n_clusters, n_clusters), \ + dtype=floating + Half pairwise distances between centers. + + distance_next_center : ndarray of shape (n_clusters,), dtype=floating + Distance between each center its closest center. + + upper_bounds : ndarray of shape (n_samples,), dtype=floating + Upper bound for the distance between each sample and its center, + updated inplace. - init : nd-array, shape (n_clusters, n_features) - Initial position of centers. + lower_bounds : ndarray of shape (n_samples, n_clusters), dtype=floating + Lower bound for the distance between each sample and each center, + updated inplace. - tol : float, default=1e-4 - The relative increment in cluster means before declaring convergence. + labels : ndarray of shape (n_samples,), dtype=int + labels assignment. - max_iter : int, default=30 - Maximum number of iterations of the k-means algorithm. + center_shift : ndarray of shape (n_clusters,), dtype=floating + Distance between old and new centers. - verbose : bool, default=False - Whether to be verbose. + n_threads : int + The number of threads to be used by openmp. + update_centers : bool + - If True, the labels and the new centers will be computed, i.e. runs + the E-step and the M-step of the algorithm. + - If False, only the labels will be computed, i.e runs the E-step of + the algorithm. This is useful especially when calling predict on a + fitted model. """ - if floating is float: - dtype = np.float32 - else: - dtype = np.float64 - - # initialize - cdef np.ndarray[floating, ndim=2, mode='c'] centers_ = init - cdef floating* centers_p = centers_.data - cdef floating* X_p = X_.data - cdef floating* x_p - cdef Py_ssize_t n_samples = X_.shape[0] - cdef Py_ssize_t n_features = X_.shape[1] - cdef Py_ssize_t point_index - cdef int center_index, label - cdef floating upper_bound, distance - cdef floating[:, :] center_half_distances = euclidean_distances(centers_) / 2. - cdef floating[:, :] lower_bounds = np.zeros((n_samples, n_clusters), dtype=dtype) - cdef floating[:] distance_next_center - labels_ = np.empty(n_samples, dtype=np.int32) - cdef int[:] labels = labels_ - upper_bounds_ = np.empty(n_samples, dtype=dtype) - cdef floating[:] upper_bounds = upper_bounds_ - - # Get the initial set of upper bounds and lower bounds for each sample. - update_labels_distances_inplace(X_p, centers_p, center_half_distances, - labels, lower_bounds, upper_bounds, - n_samples, n_features, n_clusters) - cdef np.uint8_t[:] bounds_tight = np.ones(n_samples, dtype=np.uint8) - cdef np.uint8_t[:] points_to_update = np.zeros(n_samples, dtype=np.uint8) - cdef np.ndarray[floating, ndim=2, mode='c'] new_centers - - if max_iter <= 0: - raise ValueError('Number of iterations should be a positive number' - ', got %d instead' % max_iter) - - col_indices = np.arange(center_half_distances.shape[0], dtype=np.int) - for iteration in range(max_iter): - if verbose: - print("start iteration") - - cd = np.asarray(center_half_distances) - distance_next_center = np.partition(cd, kth=1, axis=0)[1] - - if verbose: - print("done sorting") - - for point_index in range(n_samples): - upper_bound = upper_bounds[point_index] - label = labels[point_index] - - # This means that the next likely center is far away from the - # currently assigned center and the sample is unlikely to be - # reassigned. - if distance_next_center[label] >= upper_bound: - continue - x_p = X_p + point_index * n_features - - # TODO: get pointer to lower_bounds[point_index, center_index] - for center_index in range(n_clusters): + cdef: + int n_samples = X.shape[0] + int n_features = X.shape[1] + int n_clusters = centers_new.shape[0] + + # hard-coded number of samples per chunk. Splitting in chunks is + # necessary to get parallelism. Chunk size chosed to be same as lloyd's + int n_samples_chunk = 256 if n_samples > 256 else n_samples + int n_chunks = n_samples // n_samples_chunk + int n_samples_rem = n_samples % n_samples_chunk + int chunk_idx, n_samples_chunk_eff + int start, end + + int i, j, k + + floating *centers_new_chunk + floating *weight_in_clusters_chunk + + # count remainder chunk in total number of chunks + n_chunks += n_samples != n_chunks * n_samples_chunk + + if update_centers: + memset(¢ers_new[0, 0], 0, n_clusters * n_features * sizeof(floating)) + memset(&weight_in_clusters[0], 0, n_clusters * sizeof(floating)) + + with nogil, parallel(num_threads=n_threads): + # thread local buffers + centers_new_chunk = calloc(n_clusters * n_features, sizeof(floating)) + weight_in_clusters_chunk = calloc(n_clusters, sizeof(floating)) + + for chunk_idx in prange(n_chunks, schedule='static'): + start = chunk_idx * n_samples_chunk + if chunk_idx == n_chunks - 1 and n_samples_rem > 0: + end = start + n_samples_rem + else: + end = start + n_samples_chunk + + _update_chunk_dense( + &X[start, 0], + sample_weight[start: end], + centers_old, + center_half_distances, + distance_next_center, + labels[start: end], + upper_bounds[start: end], + lower_bounds[start: end], + centers_new_chunk, + weight_in_clusters_chunk, + update_centers) + + # reduction from local buffers. The gil is necessary for that to avoid + # race conditions. + if update_centers: + with gil: + for j in range(n_clusters): + weight_in_clusters[j] += weight_in_clusters_chunk[j] + for k in range(n_features): + centers_new[j, k] += centers_new_chunk[j * n_features + k] + + if update_centers: + _relocate_empty_clusters_dense(X, sample_weight, centers_old, + centers_new, weight_in_clusters, labels) + + _average_centers(centers_new, weight_in_clusters) + _center_shift(centers_old, centers_new, center_shift) + + # update lower and upper bounds + for i in range(n_samples): + upper_bounds[i] += center_shift[labels[i]] + + for j in range(n_clusters): + lower_bounds[i, j] -= center_shift[j] + if lower_bounds[i, j] < 0: + lower_bounds[i, j] = 0 + + +cdef void _update_chunk_dense( + floating *X, # IN + # expecting C alinged 2D array. XXX: Can be + # replaced by const memoryview when cython min + # version is >= 0.3 + floating[::1] sample_weight, # IN + floating[:, ::1] centers_old, # IN + floating[:, ::1] center_half_distances, # IN + floating[::1] distance_next_center, # IN + int[::1] labels, # INOUT + floating[::1] upper_bounds, # INOUT + floating[:, ::1] lower_bounds, # INOUT + floating *centers_new, # OUT + floating *weight_in_clusters, # OUT + bint update_centers) nogil: + """K-means combined EM step for one dense data chunk. + + Compute the partial contribution of a single data chunk to the labels and + centers. + """ + cdef: + int n_samples = labels.shape[0] + int n_clusters = centers_old.shape[0] + int n_features = centers_old.shape[1] + + floating upper_bound, distance + int i, j, k, label + + for i in range(n_samples): + upper_bound = upper_bounds[i] + bounds_tight = 0 + label = labels[i] + + # Next center is not far away from the currently assigned center. + # Sample might need to be assigned to another center. + if not distance_next_center[label] >= upper_bound: + + for j in range(n_clusters): # If this holds, then center_index is a good candidate for the # sample to be relabelled, and we need to confirm this by # recomputing the upper and lower bounds. - if (center_index != label - and (upper_bound > lower_bounds[point_index, center_index]) - and (upper_bound > center_half_distances[center_index, label])): - - # Recompute the upper bound by calculating the actual distance - # between the sample and label. - if not bounds_tight[point_index]: - upper_bound = euclidean_dist(x_p, centers_p + label * n_features, n_features) - lower_bounds[point_index, label] = upper_bound - bounds_tight[point_index] = 1 - - # If the condition still holds, then compute the actual distance between - # the sample and center_index. If this is still lesser than the previous - # distance, reassign labels. - if (upper_bound > lower_bounds[point_index, center_index] - or (upper_bound > center_half_distances[label, center_index])): - distance = euclidean_dist(x_p, centers_p + center_index * n_features, n_features) - lower_bounds[point_index, center_index] = distance + if (j != label + and (upper_bound > lower_bounds[i, j]) + and (upper_bound > center_half_distances[label, j])): + + # Recompute upper bound by calculating the actual distance + # between the sample and its current assigned center. + if not bounds_tight: + upper_bound = _euclidean_dense_dense( + X + i * n_features, ¢ers_old[label, 0], n_features, False) + lower_bounds[i, label] = upper_bound + bounds_tight = 1 + + # If the condition still holds, then compute the actual + # distance between the sample and center. If this is less + # than the previous distance, reassign label. + if (upper_bound > lower_bounds[i, j] + or (upper_bound > center_half_distances[label, j])): + + distance = _euclidean_dense_dense( + X + i * n_features, ¢ers_old[j, 0], n_features, False) + lower_bounds[i, j] = distance if distance < upper_bound: - label = center_index + label = j upper_bound = distance - labels[point_index] = label - upper_bounds[point_index] = upper_bound - - if verbose: - print("end inner loop") - - # compute new centers - new_centers = _centers_dense(X_, sample_weight, labels_, - n_clusters, upper_bounds_) - bounds_tight[:] = 0 - - # compute distance each center moved - center_shift = np.sqrt(np.sum((centers_ - new_centers) ** 2, axis=1)) - - # update bounds accordingly - lower_bounds = np.maximum(lower_bounds - center_shift, 0) - upper_bounds = upper_bounds + center_shift[labels_] - - # reassign centers - centers_ = new_centers - centers_p = new_centers.data - - # update between-center distances - center_half_distances = euclidean_distances(centers_) / 2. - if verbose: - print('Iteration %i, inertia %s' - % (iteration, np.sum((X_ - centers_[labels]) ** 2 * - sample_weight[:,np.newaxis]))) - center_shift_total = np.sum(center_shift) - if center_shift_total ** 2 < tol: - if verbose: - print("center shift %e within tolerance %e" - % (center_shift_total, tol)) - break - - # We need this to make sure that the labels give the same output as - # predict(X) - if center_shift_total > 0: - update_labels_distances_inplace(X_p, centers_p, center_half_distances, - labels, lower_bounds, upper_bounds, - n_samples, n_features, n_clusters) - return centers_, labels_, iteration + 1 + labels[i] = label + upper_bounds[i] = upper_bound + + if update_centers: + weight_in_clusters[label] += sample_weight[i] + for k in range(n_features): + centers_new[label * n_features + k] += X[i * n_features + k] * sample_weight[i] + + +def _elkan_iter_chunked_sparse( + X, # IN + floating[::1] sample_weight, # IN + floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # OUT + floating[::1] weight_in_clusters, # OUT + floating[:, ::1] center_half_distances, # IN + floating[::1] distance_next_center, # IN + floating[::1] upper_bounds, # INOUT + floating[:, ::1] lower_bounds, # INOUT + int[::1] labels, # INOUT + floating[::1] center_shift, # OUT + int n_threads, + bint update_centers=True): + """Single iteration of K-means Elkan algorithm with sparse input. + + Update labels and centers (inplace), for one iteration, distributed + over data chunks. + + Parameters + ---------- + X : sparse matrix of shape (n_samples, n_features) + The observations to cluster. Must be in CSR format. + + sample_weight : ndarray of shape (n_samples,), dtype=floating + The weights for each observation in X. + + centers_old : ndarray of shape (n_clusters, n_features), dtype=floating + Centers before previous iteration, placeholder for the centers after + previous iteration. + + centers_new : ndarray of shape (n_clusters, n_features), dtype=floating + Centers after previous iteration, placeholder for the new centers + computed during this iteration. + + weight_in_clusters : ndarray of shape (n_clusters,), dtype=floating + Placeholder for the sums of the weights of every observation assigned + to each center. + + center_half_distances : ndarray of shape (n_clusters, n_clusters), \ + dtype=floating + Half pairwise distances between centers. + + distance_next_center : ndarray of shape (n_clusters,), dtype=floating + Distance between each center its closest center. + + upper_bounds : ndarray of shape (n_samples,), dtype=floating + Upper bound for the distance between each sample and its center, + updated inplace. + + lower_bounds : ndarray of shape (n_samples, n_clusters), dtype=floating + Lower bound for the distance between each sample and each center, + updated inplace. + + labels : ndarray of shape (n_samples,), dtype=int + labels assignment. + + center_shift : ndarray of shape (n_clusters,), dtype=floating + Distance between old and new centers. + + n_threads : int + The number of threads to be used by openmp. + + update_centers : bool + - If True, the labels and the new centers will be computed, i.e. runs + the E-step and the M-step of the algorithm. + - If False, only the labels will be computed, i.e runs the E-step of + the algorithm. This is useful especially when calling predict on a + fitted model. + """ + cdef: + int n_samples = X.shape[0] + int n_features = X.shape[1] + int n_clusters = centers_new.shape[0] + + floating[::1] X_data = X.data + int[::1] X_indices = X.indices + int[::1] X_indptr = X.indptr + + # hard-coded number of samples per chunk. Splitting in chunks is + # necessary to get parallelism. Chunk size chosed to be same as lloyd's + int n_samples_chunk = 256 if n_samples > 256 else n_samples + int n_chunks = n_samples // n_samples_chunk + int n_samples_rem = n_samples % n_samples_chunk + int chunk_idx, n_samples_chunk_eff + int start, end + + int i, j, k + + floating[::1] centers_squared_norms = row_norms(centers_old, squared=True) + + floating *centers_new_chunk + floating *weight_in_clusters_chunk + + # count remainder chunk in total number of chunks + n_chunks += n_samples != n_chunks * n_samples_chunk + + if update_centers: + memset(¢ers_new[0, 0], 0, n_clusters * n_features * sizeof(floating)) + memset(&weight_in_clusters[0], 0, n_clusters * sizeof(floating)) + + with nogil, parallel(num_threads=n_threads): + # thread local buffers + centers_new_chunk = calloc(n_clusters * n_features, sizeof(floating)) + weight_in_clusters_chunk = calloc(n_clusters, sizeof(floating)) + + for chunk_idx in prange(n_chunks, schedule='static'): + start = chunk_idx * n_samples_chunk + if chunk_idx == n_chunks - 1 and n_samples_rem > 0: + end = start + n_samples_rem + else: + end = start + n_samples_chunk + + _update_chunk_sparse( + X_data[X_indptr[start]: X_indptr[end]], + X_indices[X_indptr[start]: X_indptr[end]], + X_indptr[start: end], + sample_weight[start: end], + centers_old, + centers_squared_norms, + center_half_distances, + distance_next_center, + labels[start: end], + upper_bounds[start: end], + lower_bounds[start: end], + centers_new_chunk, + weight_in_clusters_chunk, + update_centers) + + # reduction from local buffers. The gil is necessary for that to avoid + # race conditions. + if update_centers: + with gil: + for j in range(n_clusters): + weight_in_clusters[j] += weight_in_clusters_chunk[j] + for k in range(n_features): + centers_new[j, k] += centers_new_chunk[j * n_features + k] + + if update_centers: + _relocate_empty_clusters_sparse( + X_data, X_indices, X_indptr, sample_weight, + centers_old, centers_new, weight_in_clusters, labels) + + _average_centers(centers_new, weight_in_clusters) + _center_shift(centers_old, centers_new, center_shift) + + # update lower and upper bounds + for i in range(n_samples): + upper_bounds[i] += center_shift[labels[i]] + + for j in range(n_clusters): + lower_bounds[i, j] -= center_shift[j] + if lower_bounds[i, j] < 0: + lower_bounds[i, j] = 0 + + +cdef void _update_chunk_sparse( + floating[::1] X_data, # IN + int[::1] X_indices, # IN + int[::1] X_indptr, # IN + floating[::1] sample_weight, # IN + floating[:, ::1] centers_old, # IN + floating[::1] centers_squared_norms, # IN + floating[:, ::1] center_half_distances, # IN + floating[::1] distance_next_center, # IN + int[::1] labels, # INOUT + floating[::1] upper_bounds, # INOUT + floating[:, ::1] lower_bounds, # INOUT + floating *centers_new, # OUT + floating *weight_in_clusters, # OUT + bint update_centers) nogil: + """K-means combined EM step for one sparse data chunk. + + Compute the partial contribution of a single data chunk to the labels and + centers. + """ + cdef: + int n_samples = labels.shape[0] + int n_clusters = centers_old.shape[0] + int n_features = centers_old.shape[1] + + floating upper_bound, distance + int i, j, k, label + int s = X_indptr[0] + + for i in range(n_samples): + upper_bound = upper_bounds[i] + bounds_tight = 0 + label = labels[i] + + # Next center is not far away from the currently assigned center. + # Sample might need to be assigned to another center. + if not distance_next_center[label] >= upper_bound: + + for j in range(n_clusters): + + # If this holds, then center_index is a good candidate for the + # sample to be relabelled, and we need to confirm this by + # recomputing the upper and lower bounds. + if (j != label + and (upper_bound > lower_bounds[i, j]) + and (upper_bound > center_half_distances[label, j])): + + # Recompute upper bound by calculating the actual distance + # between the sample and its current assigned center. + if not bounds_tight: + upper_bound = _euclidean_sparse_dense( + X_data[X_indptr[i] - s: X_indptr[i + 1] - s], + X_indices[X_indptr[i] - s: X_indptr[i + 1] - s], + centers_old[label], centers_squared_norms[label], False) + lower_bounds[i, label] = upper_bound + bounds_tight = 1 + + # If the condition still holds, then compute the actual + # distance between the sample and center. If this is less + # than the previous distance, reassign label. + if (upper_bound > lower_bounds[i, j] + or (upper_bound > center_half_distances[label, j])): + distance = _euclidean_sparse_dense( + X_data[X_indptr[i] - s: X_indptr[i + 1] - s], + X_indices[X_indptr[i] - s: X_indptr[i + 1] - s], + centers_old[j], centers_squared_norms[j], False) + lower_bounds[i, j] = distance + if distance < upper_bound: + label = j + upper_bound = distance + + labels[i] = label + upper_bounds[i] = upper_bound + + if update_centers: + weight_in_clusters[label] += sample_weight[i] + for k in range(X_indptr[i] - s, X_indptr[i + 1] - s): + centers_new[label * n_features + X_indices[k]] += X_data[k] * sample_weight[i] diff --git a/sklearn/cluster/_k_means_fast.pxd b/sklearn/cluster/_k_means_fast.pxd new file mode 100644 index 0000000000000..b8dcd947f92c6 --- /dev/null +++ b/sklearn/cluster/_k_means_fast.pxd @@ -0,0 +1,23 @@ +# cython: language_level=3 + + +from cython cimport floating +cimport numpy as np + + +cdef floating _euclidean_dense_dense(floating*, floating*, int, bint) nogil + +cdef floating _euclidean_sparse_dense(floating[::1], int[::1], floating[::1], + floating, bint) nogil + +cpdef void _relocate_empty_clusters_dense( + np.ndarray[floating, ndim=2, mode='c'], floating[::1], floating[:, ::1], + floating[:, ::1], floating[::1], int[::1]) + +cpdef void _relocate_empty_clusters_sparse( + floating[::1], int[::1], int[::1], floating[::1], floating[:, ::1], + floating[:, ::1], floating[::1], int[::1]) + +cdef void _average_centers(floating[:, ::1], floating[::1]) + +cdef void _center_shift(floating[:, ::1], floating[:, ::1], floating[::1]) diff --git a/sklearn/cluster/_k_means_fast.pyx b/sklearn/cluster/_k_means_fast.pyx index 8a66f25065126..8221b2b15e356 100644 --- a/sklearn/cluster/_k_means_fast.pyx +++ b/sklearn/cluster/_k_means_fast.pyx @@ -1,4 +1,4 @@ -# cython: profile=True +# cython: profile=True, boundscheck=False, wraparound=False, cdivision=True # Profiling is enabled by default as the overhead does not seem to be # measurable on this specific use case. @@ -7,155 +7,286 @@ # Lars Buitinck # # License: BSD 3 clause -# -# cython: boundscheck=False, wraparound=False, cdivision=True -from libc.math cimport sqrt +# TODO: We still need to use ndarrays instead of typed memoryviews when using +# fused types and when the array may be read-only (for instance when it's +# provided by the user). This is fixed in cython > 0.3. + import numpy as np -import scipy.sparse as sp cimport numpy as np cimport cython from cython cimport floating +from libc.math cimport sqrt + +from ..utils.extmath import row_norms + + +np.import_array() -from ..utils.sparsefuncs_fast import assign_rows_csr -from ..utils._cython_blas cimport _dot ctypedef np.float64_t DOUBLE ctypedef np.int32_t INT -np.import_array() +cdef floating _euclidean_dense_dense( + floating* a, # IN + floating* b, # IN + int n_features, + bint squared) nogil: + """Euclidean distance between a dense and b dense""" + cdef: + int i + int n = n_features // 4 + int rem = n_features % 4 + floating result = 0 + + # We manually unroll the loop for better cache optimization. + for i in range(n): + result += ((a[0] - b[0]) * (a[0] - b[0]) + +(a[1] - b[1]) * (a[1] - b[1]) + +(a[2] - b[2]) * (a[2] - b[2]) + +(a[3] - b[3]) * (a[3] - b[3])) + a += 4; b += 4 + + for i in range(rem): + result += (a[i] - b[i]) * (a[i] - b[i]) + + return result if squared else sqrt(result) + + +def _euclidean_dense_dense_wrapper(floating[::1] a, floating[::1] b, + bint squared): + """Wrapper of _euclidean_dense_dense for testing purpose""" + return _euclidean_dense_dense(&a[0], &b[0], a.shape[0], squared) + + +cdef floating _euclidean_sparse_dense( + floating[::1] a_data, # IN + int[::1] a_indices, # IN + floating[::1] b, # IN + floating b_squared_norm, + bint squared) nogil: + """Euclidean distance between a sparse and b dense""" + cdef: + int nnz = a_indices.shape[0] + int i + floating tmp, bi + floating result = 0.0 + + for i in range(nnz): + bi = b[a_indices[i]] + tmp = a_data[i] - bi + result += tmp * tmp - bi * bi + + result += b_squared_norm + + if result < 0: result = 0.0 + + return result if squared else sqrt(result) -cpdef DOUBLE _assign_labels_array(np.ndarray[floating, ndim=2] X, - np.ndarray[floating, ndim=1] sample_weight, - np.ndarray[floating, ndim=1] x_squared_norms, - np.ndarray[floating, ndim=2] centers, - np.ndarray[INT, ndim=1] labels, - np.ndarray[floating, ndim=1] distances): - """Compute label assignment and inertia for a dense array +def _euclidean_sparse_dense_wrapper( + floating[::1] a_data, + int[::1] a_indices, + floating[::1] b, + floating b_squared_norm, + bint squared): + """Wrapper of _euclidean_sparse_dense for testing purpose""" + return _euclidean_sparse_dense( + a_data, a_indices, b, b_squared_norm, squared) - Return the inertia (sum of squared distances to the centers). + +cpdef floating _inertia_dense( + np.ndarray[floating, ndim=2, mode='c'] X, # IN + floating[::1] sample_weight, # IN + floating[:, ::1] centers, # IN + int[::1] labels): # IN + """Compute inertia for dense input data + + Sum of squared distance between each sample and its assigned center. """ cdef: - unsigned int n_clusters = centers.shape[0] - unsigned int n_features = centers.shape[1] - unsigned int n_samples = X.shape[0] - unsigned int x_stride - unsigned int center_stride - unsigned int sample_idx, center_idx, feature_idx - unsigned int store_distances = 0 - unsigned int k - np.ndarray[floating, ndim=1] center_squared_norms - # the following variables are always double cause make them floating - # does not save any memory, but makes the code much bigger - DOUBLE inertia = 0.0 - DOUBLE min_dist - DOUBLE dist - - if floating is float: - center_squared_norms = np.zeros(n_clusters, dtype=np.float32) - x_stride = X.strides[1] / sizeof(float) - center_stride = centers.strides[1] / sizeof(float) - else: - center_squared_norms = np.zeros(n_clusters, dtype=np.float64) - x_stride = X.strides[1] / sizeof(DOUBLE) - center_stride = centers.strides[1] / sizeof(DOUBLE) - - if n_samples == distances.shape[0]: - store_distances = 1 + int n_samples = X.shape[0] + int n_features = X.shape[1] + int i, j - for center_idx in range(n_clusters): - center_squared_norms[center_idx] = _dot( - n_features, ¢ers[center_idx, 0], center_stride, - ¢ers[center_idx, 0], center_stride) - - for sample_idx in range(n_samples): - min_dist = -1 - for center_idx in range(n_clusters): - dist = 0.0 - # hardcoded: minimize euclidean distance to cluster center: - # ||a - b||^2 = ||a||^2 + ||b||^2 -2 - dist += _dot(n_features, &X[sample_idx, 0], x_stride, - ¢ers[center_idx, 0], center_stride) - dist *= -2 - dist += center_squared_norms[center_idx] - dist += x_squared_norms[sample_idx] - dist *= sample_weight[sample_idx] - if min_dist == -1 or dist < min_dist: - min_dist = dist - labels[sample_idx] = center_idx - - if store_distances: - distances[sample_idx] = min_dist - inertia += min_dist + floating sq_dist = 0.0 + floating inertia = 0.0 + + for i in range(n_samples): + j = labels[i] + sq_dist = _euclidean_dense_dense(&X[i, 0], ¢ers[j, 0], + n_features, True) + inertia += sq_dist * sample_weight[i] return inertia -cpdef DOUBLE _assign_labels_csr(X, np.ndarray[floating, ndim=1] sample_weight, - np.ndarray[DOUBLE, ndim=1] x_squared_norms, - np.ndarray[floating, ndim=2] centers, - np.ndarray[INT, ndim=1] labels, - np.ndarray[floating, ndim=1] distances): - """Compute label assignment and inertia for a CSR input +cpdef floating _inertia_sparse( + X, # IN + floating[::1] sample_weight, # IN + floating[:, ::1] centers, # IN + int[::1] labels): # IN + """Compute inertia for sparse input data - Return the inertia (sum of squared distances to the centers). + Sum of squared distance between each sample and its assigned center. """ cdef: - np.ndarray[floating, ndim=1] X_data = X.data - np.ndarray[INT, ndim=1] X_indices = X.indices - np.ndarray[INT, ndim=1] X_indptr = X.indptr - unsigned int n_clusters = centers.shape[0] - unsigned int n_features = centers.shape[1] - unsigned int n_samples = X.shape[0] - unsigned int store_distances = 0 - unsigned int sample_idx, center_idx, feature_idx - unsigned int k - np.ndarray[floating, ndim=1] center_squared_norms - # the following variables are always double cause make them floating - # does not save any memory, but makes the code much bigger - DOUBLE inertia = 0.0 - DOUBLE min_dist - DOUBLE dist + floating[::1] X_data = X.data + int[::1] X_indices = X.indices + int[::1] X_indptr = X.indptr - if floating is float: - center_squared_norms = np.zeros(n_clusters, dtype=np.float32) - else: - center_squared_norms = np.zeros(n_clusters, dtype=np.float64) + int n_samples = X.shape[0] + int n_features = X.shape[1] + int i, j - if n_samples == distances.shape[0]: - store_distances = 1 + floating sq_dist = 0.0 + floating inertia = 0.0 - for center_idx in range(n_clusters): - center_squared_norms[center_idx] = _dot( - n_features, ¢ers[center_idx, 0], 1, - ¢ers[center_idx, 0], 1) - - for sample_idx in range(n_samples): - min_dist = -1 - for center_idx in range(n_clusters): - dist = 0.0 - # hardcoded: minimize euclidean distance to cluster center: - # ||a - b||^2 = ||a||^2 + ||b||^2 -2 - for k in range(X_indptr[sample_idx], X_indptr[sample_idx + 1]): - dist += centers[center_idx, X_indices[k]] * X_data[k] - dist *= -2 - dist += center_squared_norms[center_idx] - dist += x_squared_norms[sample_idx] - dist *= sample_weight[sample_idx] - if min_dist == -1 or dist < min_dist: - min_dist = dist - labels[sample_idx] = center_idx - if store_distances: - distances[sample_idx] = dist - inertia += min_dist + floating[::1] centers_squared_norms = row_norms(centers, squared=True) + + for i in range(n_samples): + j = labels[i] + sq_dist = _euclidean_sparse_dense( + X_data[X_indptr[i]: X_indptr[i + 1]], + X_indices[X_indptr[i]: X_indptr[i + 1]], + centers[j], centers_squared_norms[j], True) + inertia += sq_dist * sample_weight[i] return inertia +cpdef void _relocate_empty_clusters_dense( + np.ndarray[floating, ndim=2, mode='c'] X, # IN + floating[::1] sample_weight, # IN + floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # INOUT + floating[::1] weight_in_clusters, # INOUT + int[::1] labels): # IN + """Relocate centers which have no sample assigned to them.""" + cdef: + int[::1] empty_clusters = np.where(np.equal(weight_in_clusters, 0))[0].astype(np.int32) + int n_empty = empty_clusters.shape[0] + + if n_empty == 0: + return + + cdef: + int n_features = X.shape[1] + + floating[::1] distances = ((np.asarray(X) - np.asarray(centers_old)[labels])**2).sum(axis=1) + int[::1] far_from_centers = np.argpartition(distances, -n_empty)[:-n_empty-1:-1].astype(np.int32) + + int new_cluster_id, old_cluster_id, far_idx, idx, k + floating weight + + for idx in range(n_empty): + + new_cluster_id = empty_clusters[idx] + + far_idx = far_from_centers[idx] + weight = sample_weight[far_idx] + + old_cluster_id = labels[far_idx] + + for k in range(n_features): + centers_new[old_cluster_id, k] -= X[far_idx, k] * weight + centers_new[new_cluster_id, k] = X[far_idx, k] * weight + + weight_in_clusters[new_cluster_id] = weight + weight_in_clusters[old_cluster_id] -= weight + + +cpdef void _relocate_empty_clusters_sparse( + floating[::1] X_data, # IN + int[::1] X_indices, # IN + int[::1] X_indptr, # IN + floating[::1] sample_weight, # IN + floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # INOUT + floating[::1] weight_in_clusters, # INOUT + int[::1] labels): # IN + """Relocate centers which have no sample assigned to them.""" + cdef: + int[::1] empty_clusters = np.where(np.equal(weight_in_clusters, 0))[0].astype(np.int32) + int n_empty = empty_clusters.shape[0] + + if n_empty == 0: + return + + cdef: + int n_samples = X_indptr.shape[0] - 1 + int n_features = centers_old.shape[1] + floating x + int i, j, k + + floating[::1] distances = np.zeros(n_samples, dtype=X_data.base.dtype) + floating[::1] centers_squared_norms = row_norms(centers_old, squared=True) + + for i in range(n_samples): + j = labels[i] + distances[i] = _euclidean_sparse_dense( + X_data[X_indptr[i]: X_indptr[i + 1]], + X_indices[X_indptr[i]: X_indptr[i + 1]], + centers_old[j], centers_squared_norms[j], True) + + cdef: + int[::1] far_from_centers = np.argpartition(distances, -n_empty)[:-n_empty-1:-1].astype(np.int32) + + int new_cluster_id, old_cluster_id, far_idx, idx + floating weight + + for idx in range(n_empty): + + new_cluster_id = empty_clusters[idx] + + far_idx = far_from_centers[idx] + weight = sample_weight[far_idx] + + old_cluster_id = labels[far_idx] + + for k in range(X_indptr[far_idx], X_indptr[far_idx + 1]): + centers_new[old_cluster_id, X_indices[k]] -= X_data[k] * weight + centers_new[new_cluster_id, X_indices[k]] = X_data[k] * weight + + weight_in_clusters[new_cluster_id] = weight + weight_in_clusters[old_cluster_id] -= weight + + +cdef void _average_centers( + floating[:, ::1] centers, # INOUT + floating[::1] weight_in_clusters): # IN + """Average new centers wrt weights.""" + cdef: + int n_clusters = centers.shape[0] + int n_features = centers.shape[1] + int j, k + floating alpha + + for j in range(n_clusters): + if weight_in_clusters[j] > 0: + alpha = 1.0 / weight_in_clusters[j] + for k in range(n_features): + centers[j, k] *= alpha + + +cdef void _center_shift( + floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # IN + floating[::1] center_shift): # OUT + """Compute shift between old and new centers.""" + cdef: + int n_clusters = centers_old.shape[0] + int n_features = centers_old.shape[1] + int j + + for j in range(n_clusters): + center_shift[j] = _euclidean_dense_dense( + ¢ers_new[j, 0], ¢ers_old[j, 0], n_features, False) + + def _mini_batch_update_csr(X, np.ndarray[floating, ndim=1] sample_weight, - np.ndarray[DOUBLE, ndim=1] x_squared_norms, + np.ndarray[floating, ndim=1] x_squared_norms, np.ndarray[floating, ndim=2] centers, np.ndarray[floating, ndim=1] weight_sums, np.ndarray[INT, ndim=1] nearest_center, @@ -253,143 +384,3 @@ def _mini_batch_update_csr(X, np.ndarray[floating, ndim=1] sample_weight, - centers[center_idx, feature_idx]) ** 2 return squared_diff - - -def _centers_dense(np.ndarray[floating, ndim=2] X, - np.ndarray[floating, ndim=1] sample_weight, - np.ndarray[INT, ndim=1] labels, int n_clusters, - np.ndarray[floating, ndim=1] distances): - """M step of the K-means EM algorithm - - Computation of cluster centers / means. - - Parameters - ---------- - X : array-like, shape (n_samples, n_features) - - sample_weight : array-like, shape (n_samples,) - The weights for each observation in X. - - labels : array of integers, shape (n_samples) - Current label assignment - - n_clusters : int - Number of desired clusters - - distances : array-like, shape (n_samples) - Distance to closest cluster for each sample. - - Returns - ------- - centers : array, shape (n_clusters, n_features) - The resulting centers - """ - ## TODO: add support for CSR input - cdef int n_samples, n_features - n_samples = X.shape[0] - n_features = X.shape[1] - cdef int i, j, c - cdef np.ndarray[floating, ndim=2] centers - cdef np.ndarray[floating, ndim=1] weight_in_cluster - - dtype = np.float32 if floating is float else np.float64 - centers = np.zeros((n_clusters, n_features), dtype=dtype) - weight_in_cluster = np.zeros((n_clusters,), dtype=dtype) - - for i in range(n_samples): - c = labels[i] - weight_in_cluster[c] += sample_weight[i] - empty_clusters = np.where(weight_in_cluster == 0)[0] - # maybe also relocate small clusters? - - if len(empty_clusters): - # find points to reassign empty clusters to - far_from_centers = distances.argsort()[::-1] - - for i, cluster_id in enumerate(empty_clusters): - # XXX two relocated clusters could be close to each other - far_index = far_from_centers[i] - new_center = X[far_index] * sample_weight[far_index] - centers[cluster_id] = new_center - weight_in_cluster[cluster_id] = sample_weight[far_index] - - for i in range(n_samples): - for j in range(n_features): - centers[labels[i], j] += X[i, j] * sample_weight[i] - - centers /= weight_in_cluster[:, np.newaxis] - - return centers - - -def _centers_sparse(X, np.ndarray[floating, ndim=1] sample_weight, - np.ndarray[INT, ndim=1] labels, n_clusters, - np.ndarray[floating, ndim=1] distances): - """M step of the K-means EM algorithm - - Computation of cluster centers / means. - - Parameters - ---------- - X : scipy.sparse.csr_matrix, shape (n_samples, n_features) - - sample_weight : array-like, shape (n_samples,) - The weights for each observation in X. - - labels : array of integers, shape (n_samples) - Current label assignment - - n_clusters : int - Number of desired clusters - - distances : array-like, shape (n_samples) - Distance to closest cluster for each sample. - - Returns - ------- - centers : array, shape (n_clusters, n_features) - The resulting centers - """ - cdef int n_samples, n_features - n_samples = X.shape[0] - n_features = X.shape[1] - cdef int curr_label - - cdef np.ndarray[floating, ndim=1] data = X.data - cdef np.ndarray[int, ndim=1] indices = X.indices - cdef np.ndarray[int, ndim=1] indptr = X.indptr - - cdef np.ndarray[floating, ndim=2, mode="c"] centers - cdef np.ndarray[np.npy_intp, ndim=1] far_from_centers - cdef np.ndarray[floating, ndim=1] weight_in_cluster - dtype = np.float32 if floating is float else np.float64 - centers = np.zeros((n_clusters, n_features), dtype=dtype) - weight_in_cluster = np.zeros((n_clusters,), dtype=dtype) - for i in range(n_samples): - c = labels[i] - weight_in_cluster[c] += sample_weight[i] - cdef np.ndarray[np.npy_intp, ndim=1, mode="c"] empty_clusters = \ - np.where(weight_in_cluster == 0)[0] - cdef int n_empty_clusters = empty_clusters.shape[0] - - # maybe also relocate small clusters? - - if n_empty_clusters > 0: - # find points to reassign empty clusters to - far_from_centers = distances.argsort()[::-1][:n_empty_clusters] - - # XXX two relocated clusters could be close to each other - assign_rows_csr(X, far_from_centers, empty_clusters, centers) - - for i in range(n_empty_clusters): - weight_in_cluster[empty_clusters[i]] = 1 - - for i in range(labels.shape[0]): - curr_label = labels[i] - for ind in range(indptr[i], indptr[i + 1]): - j = indices[ind] - centers[curr_label, j] += data[ind] * sample_weight[i] - - centers /= weight_in_cluster[:, np.newaxis] - - return centers diff --git a/sklearn/cluster/_k_means_lloyd.pyx b/sklearn/cluster/_k_means_lloyd.pyx new file mode 100644 index 0000000000000..93e2c6f0b9c89 --- /dev/null +++ b/sklearn/cluster/_k_means_lloyd.pyx @@ -0,0 +1,407 @@ +# cython: profile=True, boundscheck=False, wraparound=False, cdivision=True +# +# Licence: BSD 3 clause + +# TODO: We still need to use ndarrays instead of typed memoryviews when using +# fused types and when the array may be read-only (for instance when it's +# provided by the user). This is fixed in cython > 0.3. + +import numpy as np +cimport numpy as np +from cython cimport floating +from cython.parallel import prange, parallel +from libc.stdlib cimport malloc, calloc, free +from libc.string cimport memset, memcpy +from libc.float cimport DBL_MAX, FLT_MAX + +from ..utils.extmath import row_norms +from ..utils._cython_blas cimport _gemm +from ..utils._cython_blas cimport RowMajor, Trans, NoTrans +from ._k_means_fast cimport _relocate_empty_clusters_dense +from ._k_means_fast cimport _relocate_empty_clusters_sparse +from ._k_means_fast cimport _average_centers, _center_shift + + +np.import_array() + + +def _lloyd_iter_chunked_dense( + np.ndarray[floating, ndim=2, mode='c'] X, # IN + floating[::1] sample_weight, # IN + floating[::1] x_squared_norms, # IN + floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # OUT + floating[::1] weight_in_clusters, # OUT + int[::1] labels, # OUT + floating[::1] center_shift, # OUT + int n_threads, + bint update_centers=True): + """Single iteration of K-means lloyd algorithm with dense input. + + Update labels and centers (inplace), for one iteration, distributed + over data chunks. + + Parameters + ---------- + X : ndarray of shape (n_samples, n_features), dtype=floating + The observations to cluster. + + sample_weight : ndarray of shape (n_samples,), dtype=floating + The weights for each observation in X. + + x_squared_norms : ndarray of shape (n_samples,), dtype=floating + Squared L2 norm of X. + + centers_old : ndarray of shape (n_clusters, n_features), dtype=floating + Centers before previous iteration, placeholder for the centers after + previous iteration. + + centers_new : ndarray of shape (n_clusters, n_features), dtype=floating + Centers after previous iteration, placeholder for the new centers + computed during this iteration. + + centers_squared_norms : ndarray of shape (n_clusters,), dtype=floating + Squared L2 norm of the centers. + + weight_in_clusters : ndarray of shape (n_clusters,), dtype=floating + Placeholder for the sums of the weights of every observation assigned + to each center. + + labels : ndarray of shape (n_samples,), dtype=int + labels assignment. + + center_shift : ndarray of shape (n_clusters,), dtype=floating + Distance between old and new centers. + + n_threads : int + The number of threads to be used by openmp. + + update_centers : bool + - If True, the labels and the new centers will be computed, i.e. runs + the E-step and the M-step of the algorithm. + - If False, only the labels will be computed, i.e runs the E-step of + the algorithm. This is useful especially when calling predict on a + fitted model. + """ + cdef: + int n_samples = X.shape[0] + int n_features = X.shape[1] + int n_clusters = centers_new.shape[0] + + # hard-coded number of samples per chunk. Appeared to be close to + # optimal in all situations. + int n_samples_chunk = 256 if n_samples > 256 else n_samples + int n_chunks = n_samples // n_samples_chunk + int n_samples_rem = n_samples % n_samples_chunk + int chunk_idx, n_samples_chunk_eff + int start, end + + int j, k + + floating[::1] centers_squared_norms = row_norms(centers_old, squared=True) + + floating *centers_new_chunk + floating *weight_in_clusters_chunk + floating *pairwise_distances_chunk + + # count remainder chunk in total number of chunks + n_chunks += n_samples != n_chunks * n_samples_chunk + + if update_centers: + memset(¢ers_new[0, 0], 0, n_clusters * n_features * sizeof(floating)) + memset(&weight_in_clusters[0], 0, n_clusters * sizeof(floating)) + + with nogil, parallel(num_threads=n_threads): + # thread local buffers + centers_new_chunk = calloc(n_clusters * n_features, sizeof(floating)) + weight_in_clusters_chunk = calloc(n_clusters, sizeof(floating)) + pairwise_distances_chunk = malloc(n_samples_chunk * n_clusters * sizeof(floating)) + + for chunk_idx in prange(n_chunks, schedule='static'): + start = chunk_idx * n_samples_chunk + if chunk_idx == n_chunks - 1 and n_samples_rem > 0: + end = start + n_samples_rem + else: + end = start + n_samples_chunk + + _update_chunk_dense( + &X[start, 0], + sample_weight[start: end], + x_squared_norms[start: end], + centers_old, + centers_squared_norms, + labels[start: end], + centers_new_chunk, + weight_in_clusters_chunk, + pairwise_distances_chunk, + update_centers) + + # reduction from local buffers. The gil is necessary for that to avoid + # race conditions. + if update_centers: + with gil: + for j in range(n_clusters): + weight_in_clusters[j] += weight_in_clusters_chunk[j] + for k in range(n_features): + centers_new[j, k] += centers_new_chunk[j * n_features + k] + + free(centers_new_chunk) + free(weight_in_clusters_chunk) + free(pairwise_distances_chunk) + + if update_centers: + _relocate_empty_clusters_dense(X, sample_weight, centers_old, + centers_new, weight_in_clusters, labels) + + _average_centers(centers_new, weight_in_clusters) + _center_shift(centers_old, centers_new, center_shift) + + +cdef void _update_chunk_dense( + floating *X, # IN + # expecting C alinged 2D array. XXX: Can be + # replaced by const memoryview when cython min + # version is >= 0.3 + floating[::1] sample_weight, # IN + floating[::1] x_squared_norms, # IN + floating[:, ::1] centers_old, # IN + floating[::1] centers_squared_norms, # IN + int[::1] labels, # OUT + floating *centers_new, # OUT + floating *weight_in_clusters, # OUT + floating *pairwise_distances, # OUT + bint update_centers) nogil: + """K-means combined EM step for one dense data chunk. + + Compute the partial contribution of a single data chunk to the labels and + centers. + """ + cdef: + int n_samples = labels.shape[0] + int n_clusters = centers_old.shape[0] + int n_features = centers_old.shape[1] + + floating sq_dist, min_sq_dist + int i, j, k, label + + # Instead of computing the full pairwise squared distances matrix, + # ||X - C||² = ||X||² - 2 X.C^T + ||C||², we only need to store + # the - 2 X.C^T + ||C||² term since the argmin for a given sample only + # depends on the centers. + # pairwise_distances = ||C||² + for i in range(n_samples): + for j in range(n_clusters): + pairwise_distances[i * n_clusters + j] = centers_squared_norms[j] + + # pairwise_distances += -2 * X.dot(C.T) + _gemm(RowMajor, NoTrans, Trans, n_samples, n_clusters, n_features, + -2.0, X, n_features, ¢ers_old[0, 0], n_features, + 1.0, pairwise_distances, n_clusters) + + for i in range(n_samples): + min_sq_dist = pairwise_distances[i * n_clusters] + label = 0 + for j in range(1, n_clusters): + sq_dist = pairwise_distances[i * n_clusters + j] + if sq_dist < min_sq_dist: + min_sq_dist = sq_dist + label = j + labels[i] = label + + if update_centers: + weight_in_clusters[label] += sample_weight[i] + for k in range(n_features): + centers_new[label * n_features + k] += X[i * n_features + k] * sample_weight[i] + + +def _lloyd_iter_chunked_sparse( + X, # IN + floating[::1] sample_weight, # IN + floating[::1] x_squared_norms, # IN + floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # OUT + floating[::1] weight_in_clusters, # OUT + int[::1] labels, # OUT + floating[::1] center_shift, # OUT + int n_threads, + bint update_centers=True): + """Single iteration of K-means lloyd algorithm with sparse input. + + Update labels and centers (inplace), for one iteration, distributed + over data chunks. + + Parameters + ---------- + X : sparse matrix of shape (n_samples, n_features), dtype=floating + The observations to cluster. Must be in CSR format. + + sample_weight : ndarray of shape (n_samples,), dtype=floating + The weights for each observation in X. + + x_squared_norms : ndarray of shape (n_samples,), dtype=floating + Squared L2 norm of X. + + centers_old : ndarray of shape (n_clusters, n_features), dtype=floating + Centers before previous iteration, placeholder for the centers after + previous iteration. + + centers_new : ndarray of shape (n_clusters, n_features), dtype=floating + Centers after previous iteration, placeholder for the new centers + computed during this iteration. + + centers_squared_norms : ndarray of shape (n_clusters,), dtype=floating + Squared L2 norm of the centers. + + weight_in_clusters : ndarray of shape (n_clusters,), dtype=floating + Placeholder for the sums of the weights of every observation assigned + to each center. + + labels : ndarray of shape (n_samples,), dtype=int + labels assignment. + + center_shift : ndarray of shape (n_clusters,), dtype=floating + Distance between old and new centers. + + n_threads : int + The number of threads to be used by openmp. + + update_centers : bool + - If True, the labels and the new centers will be computed, i.e. runs + the E-step and the M-step of the algorithm. + - If False, only the labels will be computed, i.e runs the E-step of + the algorithm. This is useful especially when calling predict on a + fitted model. + """ + # print(X.indices.dtype) + cdef: + int n_samples = X.shape[0] + int n_features = X.shape[1] + int n_clusters = centers_new.shape[0] + + # Chosed same as for dense. Does not have the same impact since with + # sparse data the pairwise distances matrix is not precomputed. + # However, splitting in chunks is necessary to get parallelism. + int n_samples_chunk = 256 if n_samples > 256 else n_samples + int n_chunks = n_samples // n_samples_chunk + int n_samples_rem = n_samples % n_samples_chunk + int chunk_idx, n_samples_chunk_eff = 0 + int start = 0, end = 0 + + int j, k + + floating[::1] X_data = X.data + int[::1] X_indices = X.indices + int[::1] X_indptr = X.indptr + + floating[::1] centers_squared_norms = row_norms(centers_old, squared=True) + + floating *centers_new_chunk + floating *weight_in_clusters_chunk + + # count remainder chunk in total number of chunks + n_chunks += n_samples != n_chunks * n_samples_chunk + + if update_centers: + memset(¢ers_new[0, 0], 0, n_clusters * n_features * sizeof(floating)) + memset(&weight_in_clusters[0], 0, n_clusters * sizeof(floating)) + + with nogil, parallel(num_threads=n_threads): + # thread local buffers + centers_new_chunk = calloc(n_clusters * n_features, sizeof(floating)) + weight_in_clusters_chunk = calloc(n_clusters, sizeof(floating)) + + for chunk_idx in prange(n_chunks, schedule='static'): + start = chunk_idx * n_samples_chunk + if chunk_idx == n_chunks - 1 and n_samples_rem > 0: + end = start + n_samples_rem + else: + end = start + n_samples_chunk + + _update_chunk_sparse( + X_data[X_indptr[start]: X_indptr[end]], + X_indices[X_indptr[start]: X_indptr[end]], + X_indptr[start: end], + sample_weight[start: end], + x_squared_norms[start: end], + centers_old, + centers_squared_norms, + labels[start: end], + centers_new_chunk, + weight_in_clusters_chunk, + update_centers) + + # reduction from local buffers. The gil is necessary for that to avoid + # race conditions. + if update_centers: + with gil: + for j in range(n_clusters): + weight_in_clusters[j] += weight_in_clusters_chunk[j] + for k in range(n_features): + centers_new[j, k] += centers_new_chunk[j * n_features + k] + + free(centers_new_chunk) + free(weight_in_clusters_chunk) + + if update_centers: + _relocate_empty_clusters_sparse( + X_data, X_indices, X_indptr, sample_weight, + centers_old, centers_new, weight_in_clusters, labels) + + _average_centers(centers_new, weight_in_clusters) + _center_shift(centers_old, centers_new, center_shift) + + +cdef void _update_chunk_sparse( + floating[::1] X_data, # IN + int[::1] X_indices, # IN + int[::1] X_indptr, # IN + floating[::1] sample_weight, # IN + floating[::1] x_squared_norms, # IN + floating[:, ::1] centers_old, # IN + floating[::1] centers_squared_norms, # IN + int[::1] labels, # OUT + floating *centers_new, # OUT + floating *weight_in_clusters, # OUT + bint update_centers) nogil: + """K-means combined EM step for one sparse data chunk. + + Compute the partial contribution of a single data chunk to the labels and + centers. + """ + cdef: + int n_samples = labels.shape[0] + int n_clusters = centers_old.shape[0] + int n_features = centers_old.shape[1] + + floating sq_dist, min_sq_dist + int i, j, k, label + floating max_floating = FLT_MAX if floating is float else DBL_MAX + int s = X_indptr[0] + + # XXX Precompute the pairwise distances matrix is not worth for sparse + # currently. Should be tested when BLAS (sparse x dense) matrix + # multiplication is available. + for i in range(n_samples): + min_sq_dist = max_floating + label = 0 + + for j in range(n_clusters): + sq_dist = 0.0 + for k in range(X_indptr[i] - s, X_indptr[i + 1] - s): + sq_dist += centers_old[j, X_indices[k]] * X_data[k] + + # Instead of computing the full squared distance with each cluster, + # ||X - C||² = ||X||² - 2 X.C^T + ||C||², we only need to compute + # the - 2 X.C^T + ||C||² term since the argmin for a given sample + # only depends on the centers C. + sq_dist = centers_squared_norms[j] -2 * sq_dist + if sq_dist < min_sq_dist: + min_sq_dist = sq_dist + label = j + + labels[i] = label + + if update_centers: + weight_in_clusters[label] += sample_weight[i] + for k in range(X_indptr[i] - s, X_indptr[i + 1] - s): + centers_new[label * n_features + X_indices[k]] += X_data[k] * sample_weight[i] diff --git a/sklearn/cluster/_k_means.py b/sklearn/cluster/_kmeans.py similarity index 71% rename from sklearn/cluster/_k_means.py rename to sklearn/cluster/_kmeans.py index 52f2b5fee4dac..27ec0e5f388f6 100644 --- a/sklearn/cluster/_k_means.py +++ b/sklearn/cluster/_kmeans.py @@ -15,23 +15,29 @@ import numpy as np import scipy.sparse as sp -from joblib import Parallel, delayed, effective_n_jobs +from threadpoolctl import threadpool_limits from ..base import BaseEstimator, ClusterMixin, TransformerMixin from ..metrics.pairwise import euclidean_distances -from ..metrics.pairwise import pairwise_distances_argmin_min -from ..utils.extmath import row_norms, squared_norm, stable_cumsum +from ..utils.extmath import row_norms, stable_cumsum from ..utils.sparsefuncs_fast import assign_rows_csr from ..utils.sparsefuncs import mean_variance_axis -from ..utils.validation import _num_samples +from ..utils.validation import _num_samples, _deprecate_positional_args from ..utils import check_array from ..utils import gen_batches from ..utils import check_random_state from ..utils.validation import check_is_fitted, _check_sample_weight -from ..utils.validation import FLOAT_DTYPES +from ..utils._openmp_helpers import _openmp_effective_n_threads from ..exceptions import ConvergenceWarning -from . import _k_means_fast as _k_means -from ._k_means_elkan import k_means_elkan +from ._k_means_fast import _inertia_dense +from ._k_means_fast import _inertia_sparse +from ._k_means_fast import _mini_batch_update_csr +from ._k_means_lloyd import _lloyd_iter_chunked_dense +from ._k_means_lloyd import _lloyd_iter_chunked_sparse +from ._k_means_elkan import _init_bounds_dense +from ._k_means_elkan import _init_bounds_sparse +from ._k_means_elkan import _elkan_iter_chunked_dense +from ._k_means_elkan import _elkan_iter_chunked_sparse ############################################################################### @@ -43,22 +49,21 @@ def _k_init(X, n_clusters, x_squared_norms, random_state, n_local_trials=None): Parameters ---------- - X : array or sparse matrix, shape (n_samples, n_features) + X : {ndarray, sparse matrix} of shape (n_samples, n_features) The data to pick seeds for. To avoid memory copy, the input data should be double precision (dtype=np.float64). - n_clusters : integer + n_clusters : int The number of seeds to choose - x_squared_norms : array, shape (n_samples,) + x_squared_norms : ndarray of shape (n_samples,) Squared Euclidean norm of each data point. - random_state : int, RandomState instance - The generator used to initialize the centers. Use an int to make the - randomness deterministic. + random_state : RandomState instance + The generator used to initialize the centers. See :term:`Glossary `. - n_local_trials : integer, optional + n_local_trials : int, default=None The number of seeding trials for each center (except the first), of which the one reducing inertia the most is greedily chosen. Set to None to make the number of trials depend logarithmically @@ -153,6 +158,8 @@ def _validate_center_shape(X, n_centers, centers): def _tolerance(X, tol): """Return a tolerance which is independent of the dataset""" + if tol == 0: + return 0 if sp.issparse(X): variances = mean_variance_axis(X, axis=0)[1] else: @@ -176,16 +183,16 @@ def _check_normalize_sample_weight(sample_weight, X): def k_means(X, n_clusters, sample_weight=None, init='k-means++', - precompute_distances='auto', n_init=10, max_iter=300, + precompute_distances='deprecated', n_init=10, max_iter=300, verbose=False, tol=1e-4, random_state=None, copy_x=True, - n_jobs=None, algorithm="auto", return_n_iter=False): + n_jobs='deprecated', algorithm="auto", return_n_iter=False): """K-means clustering algorithm. Read more in the :ref:`User Guide `. Parameters ---------- - X : array-like or sparse matrix, shape (n_samples, n_features) + X : {array-like, sparse} matrix of shape (n_samples, n_features) The observations to cluster. It must be noted that the data will be converted to C ordering, which will cause a memory copy if the given data is not C-contiguous. @@ -194,25 +201,25 @@ def k_means(X, n_clusters, sample_weight=None, init='k-means++', The number of clusters to form as well as the number of centroids to generate. - sample_weight : array-like, shape (n_samples,), optional + sample_weight : array-like of shape (n_samples,), default=None The weights for each observation in X. If None, all observations - are assigned equal weight (default: None) + are assigned equal weight - init : {'k-means++', 'random', or ndarray, or a callable}, optional - Method for initialization, default to 'k-means++': + init : {'k-means++', 'random', ndarray, callable}, default='k-means++' + Method for initialization: 'k-means++' : selects initial cluster centers for k-mean clustering in a smart way to speed up convergence. See section Notes in k_init for more details. - 'random': choose k observations (rows) at random from data for - the initial centroids. + 'random': choose `n_clusters` observations (rows) at random from data + for the initial centroids. If an ndarray is passed, it should be of shape (n_clusters, n_features) and gives the initial centers. - If a callable is passed, it should take arguments X, k and - and a random state and return an initialization. + If a callable is passed, it should take arguments X, n_clusters and a + random state and return an initialization. precompute_distances : {'auto', True, False} Precompute distances (faster but takes more memory). @@ -225,57 +232,73 @@ def k_means(X, n_clusters, sample_weight=None, init='k-means++', False : never precompute distances - n_init : int, optional, default: 10 + .. deprecated:: 0.23 + 'precompute_distances' was deprecated in version 0.23 and will be + removed in 0.25. It has no effect. + + n_init : int, default=10 Number of time the k-means algorithm will be run with different centroid seeds. The final results will be the best output of n_init consecutive runs in terms of inertia. - max_iter : int, optional, default 300 + max_iter : int, default=300 Maximum number of iterations of the k-means algorithm to run. - verbose : boolean, optional + verbose : bool, default=False Verbosity mode. - tol : float, optional - The relative increment in the results before declaring convergence. + tol : float, default=1e-4 + Relative tolerance with regards to Frobenius norm of the difference + in the cluster centers of two consecutive iterations to declare + convergence. + It's not advised to set `tol=0` since convergence might never be + declared due to rounding errors. Use a very small number instead. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for centroid initialization. Use an int to make the randomness deterministic. See :term:`Glossary `. - copy_x : bool, optional + copy_x : bool, default=True When pre-computing distances it is more numerically accurate to center - the data first. If copy_x is True (default), then the original data is - not modified, ensuring X is C-contiguous. If False, the original data - is modified, and put back before the function returns, but small - numerical differences may be introduced by subtracting and then adding - the data mean, in this case it will also not ensure that data is - C-contiguous which may cause a significant slowdown. - - n_jobs : int or None, optional (default=None) - The number of jobs to use for the computation. This works by computing - each of the n_init runs in parallel. - - ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. - ``-1`` means using all processors. See :term:`Glossary ` - for more details. - - algorithm : "auto", "full" or "elkan", default="auto" + the data first. If copy_x is True (default), then the original data is + not modified. If False, the original data is modified, and put back + before the function returns, but small numerical differences may be + introduced by subtracting and then adding the data mean. Note that if + the original data is not C-contiguous, a copy will be made even if + copy_x is False. If the original data is sparse, but not in CSR format, + a copy will be made even if copy_x is False. + + n_jobs : int, default=None + The number of OpenMP threads to use for the computation. Parallelism is + sample-wise on the main cython loop which assigns each sample to its + closest center. + + ``None`` or ``-1`` means using all processors. + + .. deprecated:: 0.23 + ``n_jobs`` was deprecated in version 0.23 and will be removed in + 0.25. + + algorithm : {"auto", "full", "elkan"}, default="auto" K-means algorithm to use. The classical EM-style algorithm is "full". - The "elkan" variation is more efficient by using the triangle - inequality, but currently doesn't support sparse data. "auto" chooses - "elkan" for dense data and "full" for sparse data. + The "elkan" variation is more efficient on data with well-defined + clusters, by using the triangle inequality. However it's more memory + intensive due to the allocation of an extra array of shape + (n_samples, n_clusters). + + For now "auto" (kept for backward compatibiliy) chooses "elkan" but it + might change in the future for a better heuristic. - return_n_iter : bool, optional + return_n_iter : bool, default=False Whether or not to return the number of iterations. Returns ------- - centroid : float ndarray with shape (k, n_features) + centroid : ndarray of shape (n_clusters, n_features) Centroids found at the last iteration of k-means. - label : integer ndarray with shape (n_samples,) + label : ndarray of shape (n_samples,) label[i] is the code or index of the centroid the i'th observation is closest to. @@ -287,7 +310,6 @@ def k_means(X, n_clusters, sample_weight=None, init='k-means++', Number of iterations corresponding to the best results. Returned only if `return_n_iter` is set to True. """ - est = KMeans( n_clusters=n_clusters, init=init, n_init=n_init, max_iter=max_iter, verbose=verbose, precompute_distances=precompute_distances, tol=tol, @@ -302,93 +324,69 @@ def k_means(X, n_clusters, sample_weight=None, init='k-means++', def _kmeans_single_elkan(X, sample_weight, n_clusters, max_iter=300, init='k-means++', verbose=False, x_squared_norms=None, - random_state=None, tol=1e-4, - precompute_distances=True): - if sp.issparse(X): - raise TypeError("algorithm='elkan' not supported for sparse input X") - random_state = check_random_state(random_state) - if x_squared_norms is None: - x_squared_norms = row_norms(X, squared=True) - # init - centers = _init_centroids(X, n_clusters, init, random_state=random_state, - x_squared_norms=x_squared_norms) - centers = np.ascontiguousarray(centers) - if verbose: - print('Initialization complete') - - checked_sample_weight = _check_normalize_sample_weight(sample_weight, X) - centers, labels, n_iter = k_means_elkan(X, checked_sample_weight, - n_clusters, centers, tol=tol, - max_iter=max_iter, verbose=verbose) - if sample_weight is None: - inertia = np.sum((X - centers[labels]) ** 2, dtype=np.float64) - else: - sq_distances = np.sum((X - centers[labels]) ** 2, axis=1, - dtype=np.float64) * checked_sample_weight - inertia = np.sum(sq_distances, dtype=np.float64) - return labels, inertia, centers, n_iter - - -def _kmeans_single_lloyd(X, sample_weight, n_clusters, max_iter=300, - init='k-means++', verbose=False, x_squared_norms=None, - random_state=None, tol=1e-4, - precompute_distances=True): - """A single run of k-means, assumes preparation completed prior. + random_state=None, tol=1e-4, n_threads=1): + """A single run of k-means lloyd, assumes preparation completed prior. Parameters ---------- - X : array-like of floats, shape (n_samples, n_features) - The observations to cluster. + X : {ndarray, sparse matrix} of shape (n_samples, n_features) + The observations to cluster. If sparse matrix, must be in CSR format. + + sample_weight : array-like of shape (n_samples,) + The weights for each observation in X. n_clusters : int The number of clusters to form as well as the number of centroids to generate. - sample_weight : array-like, shape (n_samples,) - The weights for each observation in X. - - max_iter : int, optional, default 300 + max_iter : int, default=300 Maximum number of iterations of the k-means algorithm to run. - init : {'k-means++', 'random', or ndarray, or a callable}, optional - Method for initialization, default to 'k-means++': + init : {'k-means++', 'random', ndarray, callable}, default='k-means++' + Method for initialization: 'k-means++' : selects initial cluster centers for k-mean clustering in a smart way to speed up convergence. See section Notes in k_init for more details. - 'random': choose k observations (rows) at random from data for - the initial centroids. - - If an ndarray is passed, it should be of shape (k, p) and gives - the initial centers. + 'random': choose `n_clusters` observations (rows) at random from data + for the initial centroids. - If a callable is passed, it should take arguments X, k and - and a random state and return an initialization. + If an ndarray is passed, it should be of shape (n_clusters, n_features) + and gives the initial centers. - tol : float, optional - The relative increment in the results before declaring convergence. + If a callable is passed, it should take arguments X, n_clusters and a + random state and return an initialization. - verbose : boolean, optional + verbose : bool, default=False Verbosity mode - x_squared_norms : array + x_squared_norms : array-like, default=None Precomputed x_squared_norms. - precompute_distances : boolean, default: True - Precompute distances (faster but takes more memory). - - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for centroid initialization. Use an int to make the randomness deterministic. See :term:`Glossary `. + tol : float, default=1e-4 + Relative tolerance with regards to Frobenius norm of the difference + in the cluster centers of two consecutive iterations to declare + convergence. + It's not advised to set `tol=0` since convergence might never be + declared due to rounding errors. Use a very small number instead. + + n_threads : int, default=1 + The number of OpenMP threads to use for the computation. Parallelism is + sample-wise on the main cython loop which assigns each sample to its + closest center. + Returns ------- - centroid : float ndarray with shape (k, n_features) + centroid : ndarray of shape (n_clusters, n_features) Centroids found at the last iteration of k-means. - label : integer ndarray with shape (n_samples,) + label : ndarray of shape (n_samples,) label[i] is the code or index of the centroid the i'th observation is closest to. @@ -400,197 +398,286 @@ def _kmeans_single_lloyd(X, sample_weight, n_clusters, max_iter=300, Number of iterations run. """ random_state = check_random_state(random_state) - sample_weight = _check_normalize_sample_weight(sample_weight, X) - best_labels, best_inertia, best_centers = None, None, None # init centers = _init_centroids(X, n_clusters, init, random_state=random_state, x_squared_norms=x_squared_norms) + if verbose: - print("Initialization complete") + print('Initialization complete') + + n_samples = X.shape[0] - # Allocate memory to store the distances for each sample to its - # closer center for reallocation in case of ties - distances = np.zeros(shape=(X.shape[0],), dtype=X.dtype) + centers_new = np.zeros_like(centers) + weight_in_clusters = np.zeros(n_clusters, dtype=X.dtype) + labels = np.full(n_samples, -1, dtype=np.int32) + center_half_distances = euclidean_distances(centers) / 2 + distance_next_center = np.partition(np.asarray(center_half_distances), + kth=1, axis=0)[1] + upper_bounds = np.zeros(n_samples, dtype=X.dtype) + lower_bounds = np.zeros((n_samples, n_clusters), dtype=X.dtype) + center_shift = np.zeros(n_clusters, dtype=X.dtype) + + if sp.issparse(X): + init_bounds = _init_bounds_sparse + elkan_iter = _elkan_iter_chunked_sparse + _inertia = _inertia_sparse + else: + init_bounds = _init_bounds_dense + elkan_iter = _elkan_iter_chunked_dense + _inertia = _inertia_dense + + init_bounds(X, centers, center_half_distances, + labels, upper_bounds, lower_bounds) - # iterations for i in range(max_iter): - centers_old = centers.copy() - # labels assignment is also called the E-step of EM - labels, inertia = \ - _labels_inertia(X, sample_weight, x_squared_norms, centers, - precompute_distances=precompute_distances, - distances=distances) - - # computation of the means is also called the M-step of EM - if sp.issparse(X): - centers = _k_means._centers_sparse(X, sample_weight, labels, - n_clusters, distances) - else: - centers = _k_means._centers_dense(X, sample_weight, labels, - n_clusters, distances) + elkan_iter(X, sample_weight, centers, centers_new, weight_in_clusters, + center_half_distances, distance_next_center, upper_bounds, + lower_bounds, labels, center_shift, n_threads) - if verbose: - print("Iteration %2d, inertia %.3f" % (i, inertia)) + # compute new pairwise distances between centers and closest other + # center of each center for next iterations + center_half_distances = euclidean_distances(centers_new) / 2 + distance_next_center = np.partition(np.asarray(center_half_distances), + kth=1, axis=0)[1] - if best_inertia is None or inertia < best_inertia: - best_labels = labels.copy() - best_centers = centers.copy() - best_inertia = inertia + if verbose: + inertia = _inertia(X, sample_weight, centers, labels) + print("Iteration {0}, inertia {1}" .format(i, inertia)) - center_shift_total = squared_norm(centers_old - centers) - if center_shift_total <= tol: + center_shift_tot = (center_shift**2).sum() + if center_shift_tot <= tol: if verbose: - print("Converged at iteration %d: " - "center shift %e within tolerance %e" - % (i, center_shift_total, tol)) + print("Converged at iteration {0}: " + "center shift {1} within tolerance {2}" + .format(i, center_shift_tot, tol)) break - if center_shift_total > 0: - # rerun E-step in case of non-convergence so that predicted labels - # match cluster centers - best_labels, best_inertia = \ - _labels_inertia(X, sample_weight, x_squared_norms, best_centers, - precompute_distances=precompute_distances, - distances=distances) + centers, centers_new = centers_new, centers + + if center_shift_tot > 0: + # rerun E-step so that predicted labels match cluster centers + elkan_iter(X, sample_weight, centers, centers, weight_in_clusters, + center_half_distances, distance_next_center, upper_bounds, + lower_bounds, labels, center_shift, n_threads, + update_centers=False) - return best_labels, best_inertia, best_centers, i + 1 + inertia = _inertia(X, sample_weight, centers, labels) + return labels, inertia, centers, i + 1 -def _labels_inertia_precompute_dense(X, sample_weight, x_squared_norms, - centers, distances): - """Compute labels and inertia using a full distance matrix. - This will overwrite the 'distances' array in-place. +def _kmeans_single_lloyd(X, sample_weight, n_clusters, max_iter=300, + init='k-means++', verbose=False, x_squared_norms=None, + random_state=None, tol=1e-4, n_threads=1): + """A single run of k-means lloyd, assumes preparation completed prior. Parameters ---------- - X : numpy array, shape (n_sample, n_features) - Input data. + X : {ndarray, sparse matrix} of shape (n_samples, n_features) + The observations to cluster. If sparse matrix, must be in CSR format. - sample_weight : array-like, shape (n_samples,) + sample_weight : ndarray of shape (n_samples,) The weights for each observation in X. - x_squared_norms : numpy array, shape (n_samples,) - Precomputed squared norms of X. + n_clusters : int + The number of clusters to form as well as the number of + centroids to generate. + + max_iter : int, default=300 + Maximum number of iterations of the k-means algorithm to run. + + init : {'k-means++', 'random', ndarray, callable}, default='k-means++' + Method for initialization: - centers : numpy array, shape (n_clusters, n_features) - Cluster centers which data is assigned to. + 'k-means++' : selects initial cluster centers for k-mean + clustering in a smart way to speed up convergence. See section + Notes in k_init for more details. - distances : numpy array, shape (n_samples,) - Pre-allocated array in which distances are stored. + 'random': choose `n_clusters` observations (rows) at random from data + for the initial centroids. + + If an ndarray is passed, it should be of shape (n_clusters, n_features) + and gives the initial centers. + + If a callable is passed, it should take arguments X, n_clusters and a + random state and return an initialization. + + verbose : bool, default=False + Verbosity mode + + x_squared_norms : ndarray of shape(n_samples,), default=None + Precomputed x_squared_norms. + + random_state : int, RandomState instance or None, default=None + Determines random number generation for centroid initialization. Use + an int to make the randomness deterministic. + See :term:`Glossary `. + + tol : float, default=1e-4 + Relative tolerance with regards to Frobenius norm of the difference + in the cluster centers of two consecutive iterations to declare + convergence. + It's not advised to set `tol=0` since convergence might never be + declared due to rounding errors. Use a very small number instead. + + n_threads : int, default=1 + The number of OpenMP threads to use for the computation. Parallelism is + sample-wise on the main cython loop which assigns each sample to its + closest center. Returns ------- - labels : numpy array, dtype=np.int, shape (n_samples,) - Indices of clusters that samples are assigned to. + centroid : ndarray of shape (n_clusters, n_features) + Centroids found at the last iteration of k-means. + + label : ndarray of shape (n_samples,) + label[i] is the code or index of the centroid the + i'th observation is closest to. inertia : float - Sum of squared distances of samples to their closest cluster center. + The final value of the inertia criterion (sum of squared distances to + the closest centroid for all observations in the training set). + n_iter : int + Number of iterations run. """ - n_samples = X.shape[0] + random_state = check_random_state(random_state) + sample_weight = _check_normalize_sample_weight(sample_weight, X) - # Breakup nearest neighbor distance computation into batches to prevent - # memory blowup in the case of a large number of samples and clusters. - # TODO: Once PR #7383 is merged use check_inputs=False in metric_kwargs. - labels, mindist = pairwise_distances_argmin_min( - X=X, Y=centers, metric='euclidean', metric_kwargs={'squared': True}) - # cython k-means code assumes int32 inputs - labels = labels.astype(np.int32, copy=False) - if n_samples == distances.shape[0]: - # distances will be changed in-place - distances[:] = mindist - inertia = (mindist * sample_weight).sum() - return labels, inertia + # init + centers = _init_centroids(X, n_clusters, init, random_state=random_state, + x_squared_norms=x_squared_norms) + + if verbose: + print("Initialization complete") + + centers_new = np.zeros_like(centers) + labels = np.full(X.shape[0], -1, dtype=np.int32) + weight_in_clusters = np.zeros(n_clusters, dtype=X.dtype) + center_shift = np.zeros(n_clusters, dtype=X.dtype) + + if sp.issparse(X): + lloyd_iter = _lloyd_iter_chunked_sparse + _inertia = _inertia_sparse + else: + lloyd_iter = _lloyd_iter_chunked_dense + _inertia = _inertia_dense + + for i in range(max_iter): + lloyd_iter(X, sample_weight, x_squared_norms, centers, centers_new, + weight_in_clusters, labels, center_shift, n_threads) + + if verbose: + inertia = _inertia(X, sample_weight, centers, labels) + print("Iteration {0}, inertia {1}" .format(i, inertia)) + center_shift_tot = (center_shift**2).sum() + if center_shift_tot <= tol: + if verbose: + print("Converged at iteration {0}: " + "center shift {1} within tolerance {2}" + .format(i, center_shift_tot, tol)) + break + + centers, centers_new = centers_new, centers + + if center_shift_tot > 0: + # rerun E-step so that predicted labels match cluster centers + lloyd_iter(X, sample_weight, x_squared_norms, centers, centers, + weight_in_clusters, labels, center_shift, n_threads, + update_centers=False) + + inertia = _inertia(X, sample_weight, centers, labels) + + return labels, inertia, centers, i + 1 -def _labels_inertia(X, sample_weight, x_squared_norms, centers, - precompute_distances=True, distances=None): + +def _labels_inertia(X, sample_weight, x_squared_norms, centers, n_threads=1): """E step of the K-means EM algorithm. Compute the labels and the inertia of the given samples and centers. - This will compute the distances in-place. Parameters ---------- - X : float64 array-like or CSR sparse matrix, shape (n_samples, n_features) - The input samples to assign to the labels. + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples to assign to the labels. If sparse matrix, must be in + CSR format. - sample_weight : array-like, shape (n_samples,) + sample_weight : array-like of shape (n_samples,) The weights for each observation in X. - x_squared_norms : array, shape (n_samples,) + x_squared_norms : ndarray of shape (n_samples,) Precomputed squared euclidean norm of each data point, to speed up computations. - centers : float array, shape (k, n_features) + centers : ndarray, shape (n_clusters, n_features) The cluster centers. - precompute_distances : boolean, default: True - Precompute distances (faster but takes more memory). - - distances : float array, shape (n_samples,) - Pre-allocated array to be filled in with each sample's distance - to the closest center. + n_threads : int, default=1 + The number of OpenMP threads to use for the computation. Parallelism is + sample-wise on the main cython loop which assigns each sample to its + closest center. Returns ------- - labels : int array of shape(n) + labels : ndarray of shape (n_samples,) The resulting assignment inertia : float Sum of squared distances of samples to their closest cluster center. """ n_samples = X.shape[0] + n_clusters = centers.shape[0] + sample_weight = _check_normalize_sample_weight(sample_weight, X) - # set the default value of centers to -1 to be able to detect any anomaly - # easily - labels = np.full(n_samples, -1, np.int32) - if distances is None: - distances = np.zeros(shape=(0,), dtype=X.dtype) - # distances will be changed in-place + labels = np.full(n_samples, -1, dtype=np.int32) + weight_in_clusters = np.zeros(n_clusters, dtype=centers.dtype) + center_shift = np.zeros_like(weight_in_clusters) + if sp.issparse(X): - inertia = _k_means._assign_labels_csr( - X, sample_weight, x_squared_norms, centers, labels, - distances=distances) + _labels = _lloyd_iter_chunked_sparse + _inertia = _inertia_sparse else: - if precompute_distances: - return _labels_inertia_precompute_dense(X, sample_weight, - x_squared_norms, centers, - distances) - inertia = _k_means._assign_labels_array( - X, sample_weight, x_squared_norms, centers, labels, - distances=distances) + _labels = _lloyd_iter_chunked_dense + _inertia = _inertia_dense + + _labels(X, sample_weight, x_squared_norms, centers, centers, + weight_in_clusters, labels, center_shift, n_threads, + update_centers=False) + + inertia = _inertia(X, sample_weight, centers, labels) + return labels, inertia -def _init_centroids(X, k, init, random_state=None, x_squared_norms=None, - init_size=None): +def _init_centroids(X, n_clusters=8, init="k-means++", random_state=None, + x_squared_norms=None, init_size=None): """Compute the initial centroids Parameters ---------- - X : array, shape (n_samples, n_features) + X : {ndarray, spare matrix} of shape (n_samples, n_features) + The input samples. - k : int - number of centroids + n_clusters : int, default=8 + number of centroids. - init : {'k-means++', 'random' or ndarray or callable} optional - Method for initialization + init : {'k-means++', 'random', ndarray, callable}, default="k-means++" + Method for initialization. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for centroid initialization. Use an int to make the randomness deterministic. See :term:`Glossary `. - x_squared_norms : array, shape (n_samples,), optional + x_squared_norms : ndarray of shape (n_samples,), default=None Squared euclidean norm of each data point. Pass it if you have it at hands already to avoid it being recomputed here. Default: None - init_size : int, optional + init_size : int, default=None Number of samples to randomly sample for speeding up the initialization (sometimes at the expense of accuracy): the only algorithm is initialized by running a batch KMeans on a @@ -598,7 +685,7 @@ def _init_centroids(X, k, init, random_state=None, x_squared_norms=None, Returns ------- - centers : array, shape(k, n_features) + centers : array of shape(k, n_features) """ random_state = check_random_state(random_state) n_samples = X.shape[0] @@ -607,32 +694,33 @@ def _init_centroids(X, k, init, random_state=None, x_squared_norms=None, x_squared_norms = row_norms(X, squared=True) if init_size is not None and init_size < n_samples: - if init_size < k: + if init_size < n_clusters: warnings.warn( "init_size=%d should be larger than k=%d. " - "Setting it to 3*k" % (init_size, k), + "Setting it to 3*k" % (init_size, n_clusters), RuntimeWarning, stacklevel=2) - init_size = 3 * k + init_size = 3 * n_clusters init_indices = random_state.randint(0, n_samples, init_size) X = X[init_indices] x_squared_norms = x_squared_norms[init_indices] n_samples = X.shape[0] - elif n_samples < k: + elif n_samples < n_clusters: raise ValueError( - "n_samples=%d should be larger than k=%d" % (n_samples, k)) + "n_samples={} should be larger than n_clusters={}" + .format(n_samples, n_clusters)) if isinstance(init, str) and init == 'k-means++': - centers = _k_init(X, k, random_state=random_state, + centers = _k_init(X, n_clusters, random_state=random_state, x_squared_norms=x_squared_norms) elif isinstance(init, str) and init == 'random': - seeds = random_state.permutation(n_samples)[:k] + seeds = random_state.permutation(n_samples)[:n_clusters] centers = X[seeds] elif hasattr(init, '__array__'): # ensure that the centers have the same dtype as X # this is a requirement of fused types of cython centers = np.array(init, dtype=X.dtype) elif callable(init): - centers = init(X, k, random_state=random_state) + centers = init(X, n_clusters, random_state=random_state) centers = np.asarray(centers, dtype=X.dtype) else: raise ValueError("the init parameter for the k-means should " @@ -642,7 +730,7 @@ def _init_centroids(X, k, init, random_state=None, x_squared_norms=None, if sp.issparse(centers): centers = centers.toarray() - _validate_center_shape(X, k, centers) + _validate_center_shape(X, n_clusters, centers) return centers @@ -654,36 +742,43 @@ class KMeans(TransformerMixin, ClusterMixin, BaseEstimator): Parameters ---------- - n_clusters : int, optional, default: 8 + n_clusters : int, default=8 The number of clusters to form as well as the number of centroids to generate. - init : {'k-means++', 'random' or an ndarray} - Method for initialization, defaults to 'k-means++': + init : {'k-means++', 'random', ndarray, callable}, default='k-means++' + Method for initialization: 'k-means++' : selects initial cluster centers for k-mean clustering in a smart way to speed up convergence. See section Notes in k_init for more details. - 'random': choose k observations (rows) at random from data for - the initial centroids. + 'random': choose `n_clusters` observations (rows) at random from data + for the initial centroids. If an ndarray is passed, it should be of shape (n_clusters, n_features) and gives the initial centers. - n_init : int, default: 10 + If a callable is passed, it should take arguments X, n_clusters and a + random state and return an initialization. + + n_init : int, default=10 Number of time the k-means algorithm will be run with different centroid seeds. The final results will be the best output of n_init consecutive runs in terms of inertia. - max_iter : int, default: 300 + max_iter : int, default=300 Maximum number of iterations of the k-means algorithm for a single run. - tol : float, default: 1e-4 - Relative tolerance with regards to inertia to declare convergence. + tol : float, default=1e-4 + Relative tolerance with regards to Frobenius norm of the difference + in the cluster centers of two consecutive iterations to declare + convergence. + It's not advised to set `tol=0` since convergence might never be + declared due to rounding errors. Use a very small number instead. - precompute_distances : {'auto', True, False} + precompute_distances : {'auto', True, False}, default='auto' Precompute distances (faster but takes more memory). 'auto' : do not precompute distances if n_samples * n_clusters > 12 @@ -694,45 +789,57 @@ class KMeans(TransformerMixin, ClusterMixin, BaseEstimator): False : never precompute distances. - verbose : int, default 0 + .. deprecated:: 0.23 + 'precompute_distances' was deprecated in version 0.22 and will be + removed in 0.25. It has no effect. + + verbose : int, default=0 Verbosity mode. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for centroid initialization. Use an int to make the randomness deterministic. See :term:`Glossary `. - copy_x : bool, optional + copy_x : bool, default=True When pre-computing distances it is more numerically accurate to center - the data first. If copy_x is True (default), then the original data is - not modified, ensuring X is C-contiguous. If False, the original data - is modified, and put back before the function returns, but small - numerical differences may be introduced by subtracting and then adding - the data mean, in this case it will also not ensure that data is - C-contiguous which may cause a significant slowdown. - - n_jobs : int or None, optional (default=None) - The number of jobs to use for the computation. This works by computing - each of the n_init runs in parallel. - - ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. - ``-1`` means using all processors. See :term:`Glossary ` - for more details. - - algorithm : "auto", "full" or "elkan", default="auto" + the data first. If copy_x is True (default), then the original data is + not modified. If False, the original data is modified, and put back + before the function returns, but small numerical differences may be + introduced by subtracting and then adding the data mean. Note that if + the original data is not C-contiguous, a copy will be made even if + copy_x is False. If the original data is sparse, but not in CSR format, + a copy will be made even if copy_x is False. + + n_jobs : int, default=None + The number of OpenMP threads to use for the computation. Parallelism is + sample-wise on the main cython loop which assigns each sample to its + closest center. + + ``None`` or ``-1`` means using all processors. + + .. deprecated:: 0.23 + ``n_jobs`` was deprecated in version 0.23 and will be removed in + 0.25. + + algorithm : {"auto", "full", "elkan"}, default="auto" K-means algorithm to use. The classical EM-style algorithm is "full". - The "elkan" variation is more efficient by using the triangle - inequality, but currently doesn't support sparse data. "auto" chooses - "elkan" for dense data and "full" for sparse data. + The "elkan" variation is more efficient on data with well-defined + clusters, by using the triangle inequality. However it's more memory + intensive due to the allocation of an extra array of shape + (n_samples, n_clusters). + + For now "auto" (kept for backward compatibiliy) chooses "elkan" but it + might change in the future for a better heuristic. Attributes ---------- - cluster_centers_ : array, [n_clusters, n_features] + cluster_centers_ : ndarray of shape (n_clusters, n_features) Coordinates of cluster centers. If the algorithm stops before fully converging (see ``tol`` and ``max_iter``), these will not be consistent with ``labels_``. - labels_ : array, shape (n_samples,) + labels_ : ndarray of shape (n_samples,) Labels of each point inertia_ : float @@ -741,7 +848,7 @@ class KMeans(TransformerMixin, ClusterMixin, BaseEstimator): n_iter_ : int Number of iterations run. - See Also + See also -------- MiniBatchKMeans @@ -788,11 +895,11 @@ class KMeans(TransformerMixin, ClusterMixin, BaseEstimator): array([[10., 2.], [ 1., 2.]]) """ - - def __init__(self, n_clusters=8, init='k-means++', n_init=10, - max_iter=300, tol=1e-4, precompute_distances='auto', + @_deprecate_positional_args + def __init__(self, n_clusters=8, *, init='k-means++', n_init=10, + max_iter=300, tol=1e-4, precompute_distances='deprecated', verbose=0, random_state=None, copy_x=True, - n_jobs=None, algorithm='auto'): + n_jobs='deprecated', algorithm='auto'): self.n_clusters = n_clusters self.init = init @@ -807,7 +914,8 @@ def __init__(self, n_clusters=8, init='k-means++', n_init=10, self.algorithm = algorithm def _check_test_data(self, X): - X = check_array(X, accept_sparse='csr', dtype=FLOAT_DTYPES) + X = check_array(X, accept_sparse='csr', dtype=[np.float64, np.float32], + order='C', accept_large_sparse=False) n_samples, n_features = X.shape expected_n_features = self.cluster_centers_.shape[1] if not n_features == expected_n_features: @@ -822,17 +930,19 @@ def fit(self, X, y=None, sample_weight=None): Parameters ---------- - X : array-like or sparse matrix, shape=(n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training instances to cluster. It must be noted that the data will be converted to C ordering, which will cause a memory copy if the given data is not C-contiguous. + If a sparse matrix is passed, a copy will be made if it's not in + CSR format. y : Ignored Not used, present here for API consistency by convention. - sample_weight : array-like, shape (n_samples,), optional + sample_weight : array-like of shape (n_samples,), default=None The weights for each observation in X. If None, all observations - are assigned equal weight (default: None). + are assigned equal weight. Returns ------- @@ -841,6 +951,19 @@ def fit(self, X, y=None, sample_weight=None): """ random_state = check_random_state(self.random_state) + if self.precompute_distances != 'deprecated': + warnings.warn("'precompute_distances' was deprecated in version " + "0.23 and will be removed in 0.25. It has no " + "effect", FutureWarning) + + if self.n_jobs != 'deprecated': + warnings.warn("'n_jobs' was deprecated in version 0.23 and will be" + " removed in 0.25.", FutureWarning) + self._n_threads = self.n_jobs + else: + self._n_threads = None + self._n_threads = _openmp_effective_n_threads(self._n_threads) + n_init = self.n_init if n_init <= 0: raise ValueError("Invalid number of initializations." @@ -852,10 +975,10 @@ def fit(self, X, y=None, sample_weight=None): ' got %d instead' % self.max_iter ) - # avoid forcing order when copy_x=False - order = "C" if self.copy_x else None - X = check_array(X, accept_sparse='csr', dtype=[np.float64, np.float32], - order=order, copy=self.copy_x) + X = self._validate_data(X, accept_sparse='csr', + dtype=[np.float64, np.float32], + order='C', copy=self.copy_x, + accept_large_sparse=False) # verify that the number of samples given is larger than k if _num_samples(X) < self.n_clusters: raise ValueError("n_samples=%d should be >= n_clusters=%d" % ( @@ -863,28 +986,10 @@ def fit(self, X, y=None, sample_weight=None): tol = _tolerance(X, self.tol) - # If the distances are precomputed every job will create a matrix of - # shape (n_clusters, n_samples). To stop KMeans from eating up memory - # we only activate this if the created matrix is guaranteed to be - # under 100MB. 12 million entries consume a little under 100MB if they - # are of type double. - precompute_distances = self.precompute_distances - if precompute_distances == 'auto': - n_samples = X.shape[0] - precompute_distances = (self.n_clusters * n_samples) < 12e6 - elif isinstance(precompute_distances, bool): - pass - else: - raise ValueError( - "precompute_distances should be 'auto' or True/False" - ", but a value of %r was passed" % - precompute_distances - ) - # Validate init array init = self.init if hasattr(init, '__array__'): - init = check_array(init, dtype=X.dtype.type, copy=True) + init = check_array(init, dtype=X.dtype.type, copy=True, order='C') _validate_center_shape(X, self.n_clusters, init) if n_init != 1: @@ -907,59 +1012,43 @@ def fit(self, X, y=None, sample_weight=None): x_squared_norms = row_norms(X, squared=True) best_labels, best_inertia, best_centers = None, None, None + algorithm = self.algorithm - if self.n_clusters == 1: - # elkan doesn't make sense for a single cluster, full will produce - # the right result. + if algorithm == "elkan" and self.n_clusters == 1: + warnings.warn("algorithm='elkan' doesn't make sense for a single " + "cluster. Using 'full' instead.", RuntimeWarning) algorithm = "full" + if algorithm == "auto": - algorithm = "full" if sp.issparse(X) else 'elkan' + algorithm = "full" if self.n_clusters == 1 else "elkan" + if algorithm == "full": kmeans_single = _kmeans_single_lloyd elif algorithm == "elkan": kmeans_single = _kmeans_single_elkan else: raise ValueError("Algorithm must be 'auto', 'full' or 'elkan', got" - " %s" % str(algorithm)) + " {}".format(str(algorithm))) + # seeds for the initializations of the kmeans runs. seeds = random_state.randint(np.iinfo(np.int32).max, size=n_init) - if effective_n_jobs(self.n_jobs) == 1: - # For a single thread, less memory is needed if we just store one - # set of the best results (as opposed to one set per run per - # thread). + + # limit number of threads in second level of nested parallelism + # (i.e. BLAS) to avoid oversubsciption. + with threadpool_limits(limits=1, user_api="blas"): for seed in seeds: # run a k-means once labels, inertia, centers, n_iter_ = kmeans_single( - X, sample_weight, self.n_clusters, - max_iter=self.max_iter, init=init, verbose=self.verbose, - precompute_distances=precompute_distances, tol=tol, - x_squared_norms=x_squared_norms, random_state=seed) + X, sample_weight, self.n_clusters, max_iter=self.max_iter, + init=init, verbose=self.verbose, tol=tol, + x_squared_norms=x_squared_norms, random_state=seed, + n_threads=self._n_threads) # determine if these results are the best so far if best_inertia is None or inertia < best_inertia: best_labels = labels.copy() best_centers = centers.copy() best_inertia = inertia best_n_iter = n_iter_ - else: - # parallelisation of k-means runs - results = Parallel(n_jobs=self.n_jobs, verbose=0)( - delayed(kmeans_single)( - X, sample_weight, self.n_clusters, - max_iter=self.max_iter, init=init, - verbose=self.verbose, tol=tol, - precompute_distances=precompute_distances, - x_squared_norms=x_squared_norms, - # Change seed to ensure variety - random_state=seed - ) - for seed in seeds) - # Get results with the lowest inertia - labels, inertia, centers, n_iters = zip(*results) - best = np.argmin(inertia) - best_labels = labels[best] - best_inertia = inertia[best] - best_centers = centers[best] - best_n_iter = n_iters[best] if not sp.issparse(X): if not self.copy_x: @@ -972,8 +1061,7 @@ def fit(self, X, y=None, sample_weight=None): "Number of distinct clusters ({}) found smaller than " "n_clusters ({}). Possibly due to duplicate points " "in X.".format(distinct_clusters, self.n_clusters), - ConvergenceWarning, stacklevel=2 - ) + ConvergenceWarning, stacklevel=2) self.cluster_centers_ = best_centers self.labels_ = best_labels @@ -995,13 +1083,13 @@ def fit_predict(self, X, y=None, sample_weight=None): y : Ignored Not used, present here for API consistency by convention. - sample_weight : array-like, shape (n_samples,), optional + sample_weight : array-like of shape (n_samples,), default=None The weights for each observation in X. If None, all observations - are assigned equal weight (default: None). + are assigned equal weight. Returns ------- - labels : array, shape [n_samples,] + labels : ndarray of shape (n_samples,) Index of the cluster each sample belongs to. """ return self.fit(X, sample_weight=sample_weight).labels_ @@ -1019,13 +1107,13 @@ def fit_transform(self, X, y=None, sample_weight=None): y : Ignored Not used, present here for API consistency by convention. - sample_weight : array-like, shape (n_samples,), optional + sample_weight : array-like of shape (n_samples,), default=None The weights for each observation in X. If None, all observations - are assigned equal weight (default: None). + are assigned equal weight. Returns ------- - X_new : array, shape [n_samples, k] + X_new : array of shape (n_samples, n_clusters) X transformed in the new space. """ # Currently, this just skips a copy of the data if it is not in @@ -1048,7 +1136,7 @@ def transform(self, X): Returns ------- - X_new : array, shape [n_samples, k] + X_new : ndarray of shape (n_samples, n_clusters) X transformed in the new space. """ check_is_fitted(self) @@ -1072,21 +1160,22 @@ def predict(self, X, sample_weight=None): X : {array-like, sparse matrix} of shape (n_samples, n_features) New data to predict. - sample_weight : array-like, shape (n_samples,), optional + sample_weight : array-like of shape (n_samples,), default=None The weights for each observation in X. If None, all observations - are assigned equal weight (default: None). + are assigned equal weight. Returns ------- - labels : array, shape [n_samples,] + labels : ndarray of shape (n_samples,) Index of the cluster each sample belongs to. """ check_is_fitted(self) X = self._check_test_data(X) x_squared_norms = row_norms(X, squared=True) + return _labels_inertia(X, sample_weight, x_squared_norms, - self.cluster_centers_)[0] + self.cluster_centers_, self._n_threads)[0] def score(self, X, y=None, sample_weight=None): """Opposite of the value of X on the K-means objective. @@ -1099,9 +1188,9 @@ def score(self, X, y=None, sample_weight=None): y : Ignored Not used, present here for API consistency by convention. - sample_weight : array-like, shape (n_samples,), optional + sample_weight : array-like of shape (n_samples,), default=None The weights for each observation in X. If None, all observations - are assigned equal weight (default: None). + are assigned equal weight. Returns ------- @@ -1112,6 +1201,7 @@ def score(self, X, y=None, sample_weight=None): X = self._check_test_data(X) x_squared_norms = row_norms(X, squared=True) + return -_labels_inertia(X, sample_weight, x_squared_norms, self.cluster_centers_)[1] @@ -1147,7 +1237,7 @@ def _mini_batch_step(X, sample_weight, x_squared_norms, centers, weight_sums, the distances of each sample to its closest center. May not be None when random_reassign is True. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for centroid initialization and to pick new clusters amongst observations with uniform probability. Use an int to make the randomness deterministic. @@ -1184,8 +1274,7 @@ def _mini_batch_step(X, sample_weight, x_squared_norms, centers, weight_sums, """ # Perform label assignment to nearest centers nearest_center, inertia = _labels_inertia(X, sample_weight, - x_squared_norms, centers, - distances=distances) + x_squared_norms, centers) if random_reassign and reassignment_ratio > 0: random_state = check_random_state(random_state) @@ -1220,7 +1309,7 @@ def _mini_batch_step(X, sample_weight, x_squared_norms, centers, weight_sums, # implementation for the sparse CSR representation completely written in # cython if sp.issparse(X): - return inertia, _k_means._mini_batch_update_csr( + return inertia, _mini_batch_update_csr( X, sample_weight, x_squared_norms, centers, weight_sums, nearest_center, old_center_buffer, compute_squared_diff) @@ -1335,12 +1424,13 @@ class MiniBatchKMeans(KMeans): Parameters ---------- - n_clusters : int, optional, default: 8 + n_clusters : int, default=8 The number of clusters to form as well as the number of centroids to generate. - init : {'k-means++', 'random' or an ndarray}, default: 'k-means++' - Method for initialization, defaults to 'k-means++': + init : {'k-means++', 'random'} or ndarray of shape \ + (n_clusters, n_features), default='k-means++' + Method for initialization 'k-means++' : selects initial cluster centers for k-mean clustering in a smart way to speed up convergence. See section @@ -1352,26 +1442,26 @@ class MiniBatchKMeans(KMeans): If an ndarray is passed, it should be of shape (n_clusters, n_features) and gives the initial centers. - max_iter : int, optional + max_iter : int, default=100 Maximum number of iterations over the complete dataset before stopping independently of any early stopping criterion heuristics. - batch_size : int, optional, default: 100 + batch_size : int, default=100 Size of the mini batches. - verbose : bool, optional + verbose : int, default=0 Verbosity mode. compute_labels : bool, default=True Compute label assignment and inertia for the complete dataset once the minibatch optimization has converged in fit. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for centroid initialization and random reassignment. Use an int to make the randomness deterministic. See :term:`Glossary `. - tol : float, default: 0.0 + tol : float, default=0.0 Control early stopping based on the relative center changes as measured by a smoothed, variance-normalized of the mean center squared position changes. This early stopping heuristics is @@ -1382,25 +1472,27 @@ class MiniBatchKMeans(KMeans): To disable convergence detection based on normalized center change, set tol to 0.0 (default). - max_no_improvement : int, default: 10 + max_no_improvement : int, default=10 Control early stopping based on the consecutive number of mini batches that does not yield an improvement on the smoothed inertia. To disable convergence detection based on inertia, set max_no_improvement to None. - init_size : int, optional, default: 3 * batch_size + init_size : int, default=None Number of samples to randomly sample for speeding up the initialization (sometimes at the expense of accuracy): the only algorithm is initialized by running a batch KMeans on a random subset of the data. This needs to be larger than n_clusters. + If `None`, `init_size= 3 * batch_size`. + n_init : int, default=3 Number of random initializations that are tried. In contrast to KMeans, the algorithm is only run once, using the best of the ``n_init`` initializations as measured by inertia. - reassignment_ratio : float, default: 0.01 + reassignment_ratio : float, default=0.01 Control the fraction of the maximum number of counts for a center to be reassigned. A higher value means that low count centers are more easily reassigned, which means that the @@ -1410,10 +1502,10 @@ class MiniBatchKMeans(KMeans): Attributes ---------- - cluster_centers_ : array, [n_clusters, n_features] + cluster_centers_ : ndarray of shape (n_clusters, n_features) Coordinates of cluster centers - labels_ : + labels_ : int Labels of each point (if compute_labels is set to True). inertia_ : float @@ -1463,8 +1555,8 @@ class MiniBatchKMeans(KMeans): >>> kmeans.predict([[0, 0], [4, 4]]) array([1, 0], dtype=int32) """ - - def __init__(self, n_clusters=8, init='k-means++', max_iter=100, + @_deprecate_positional_args + def __init__(self, n_clusters=8, *, init='k-means++', max_iter=100, batch_size=100, verbose=0, compute_labels=True, random_state=None, tol=0.0, max_no_improvement=10, init_size=None, n_init=3, reassignment_ratio=0.01): @@ -1501,8 +1593,8 @@ def fit(self, X, y=None, sample_weight=None): self """ random_state = check_random_state(self.random_state) - X = check_array(X, accept_sparse="csr", order='C', - dtype=[np.float64, np.float32]) + X = self._validate_data(X, accept_sparse="csr", order='C', + dtype=[np.float64, np.float32]) n_samples, n_features = X.shape if n_samples < self.n_clusters: raise ValueError("n_samples=%d should be >= n_clusters=%d" @@ -1721,6 +1813,13 @@ def partial_fit(self, X, y=None, sample_weight=None): 10 * (1 + self.counts_.min())) == 0 distances = np.zeros(X.shape[0], dtype=X.dtype) + # Raise error if partial_fit called on data with different number + # of features. + if X.shape[1] != self.cluster_centers_.shape[1]: + raise ValueError( + "Number of features %d does not match previous " + "data %d." % (X.shape[1], self.cluster_centers_.shape[1])) + _mini_batch_step(X, sample_weight, x_squared_norms, self.cluster_centers_, self.counts_, np.zeros(0, dtype=X.dtype), 0, diff --git a/sklearn/cluster/_mean_shift.py b/sklearn/cluster/_mean_shift.py index 0b1a1f99c26de..32dd1d3ad4fe8 100644 --- a/sklearn/cluster/_mean_shift.py +++ b/sklearn/cluster/_mean_shift.py @@ -19,7 +19,7 @@ from joblib import Parallel, delayed from collections import defaultdict -from ..utils.validation import check_is_fitted +from ..utils.validation import check_is_fitted, _deprecate_positional_args from ..utils import check_random_state, gen_batches, check_array from ..base import BaseEstimator, ClusterMixin from ..neighbors import NearestNeighbors @@ -38,20 +38,20 @@ def estimate_bandwidth(X, quantile=0.3, n_samples=None, random_state=0, X : array-like of shape (n_samples, n_features) Input points. - quantile : float, default 0.3 + quantile : float, default=0.3 should be between [0, 1] 0.5 means that the median of all pairwise distances is used. - n_samples : int, optional + n_samples : int, default=None The number of samples to use. If not given, all samples are used. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None The generator used to randomly select the samples from input points for bandwidth estimation. Use an int to make the randomness deterministic. See :term:`Glossary `. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of parallel jobs to run for neighbors search. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` @@ -119,7 +119,7 @@ def mean_shift(X, bandwidth=None, seeds=None, bin_seeding=False, X : array-like of shape (n_samples, n_features) Input data. - bandwidth : float, optional + bandwidth : float, default=None Kernel bandwidth. If bandwidth is not given, it is determined using a heuristic based on @@ -144,16 +144,16 @@ def mean_shift(X, bandwidth=None, seeds=None, bin_seeding=False, To speed up the algorithm, accept only those bins with at least min_bin_freq points as seeds. - cluster_all : boolean, default True + cluster_all : bool, default=True If true, then all points are clustered, even those orphans that are not within any kernel. Orphans are assigned to the nearest kernel. If false, then orphans are given cluster label -1. - max_iter : int, default 300 + max_iter : int, default=300 Maximum number of iterations, per seed point before the clustering operation terminates (for that seed point), if has not converged yet. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of jobs to use for the computation. This works by computing each of the n_init runs in parallel. @@ -206,7 +206,7 @@ def get_bin_seeds(X, bin_size, min_bin_freq=1): not sure how to set this, set it to the value of the bandwidth used in clustering.mean_shift. - min_bin_freq : integer, optional + min_bin_freq : int, default=1 Only bins with at least min_bin_freq will be selected as seeds. Raising this value decreases the number of seeds found, which makes mean_shift computationally cheaper. @@ -249,38 +249,38 @@ class MeanShift(ClusterMixin, BaseEstimator): Parameters ---------- - bandwidth : float, optional + bandwidth : float, default=None Bandwidth used in the RBF kernel. If not given, the bandwidth is estimated using sklearn.cluster.estimate_bandwidth; see the documentation for that function for hints on scalability (see also the Notes, below). - seeds : array, shape=[n_samples, n_features], optional + seeds : array-like of shape (n_samples, n_features), default=None Seeds used to initialize kernels. If not set, the seeds are calculated by clustering.get_bin_seeds with bandwidth as the grid size and default values for other parameters. - bin_seeding : boolean, optional + bin_seeding : bool, default=False If true, initial kernel locations are not locations of all points, but rather the location of the discretized version of points, where points are binned onto a grid whose coarseness corresponds to the bandwidth. Setting this option to True will speed up the algorithm because fewer seeds will be initialized. - default value: False + The default value is False. Ignored if seeds argument is not None. - min_bin_freq : int, optional + min_bin_freq : int, default=1 To speed up the algorithm, accept only those bins with at least - min_bin_freq points as seeds. If not defined, set to 1. + min_bin_freq points as seeds. - cluster_all : boolean, default True + cluster_all : bool, default=True If true, then all points are clustered, even those orphans that are not within any kernel. Orphans are assigned to the nearest kernel. If false, then orphans are given cluster label -1. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of jobs to use for the computation. This works by computing each of the n_init runs in parallel. @@ -299,7 +299,7 @@ class MeanShift(ClusterMixin, BaseEstimator): cluster_centers_ : array, [n_clusters, n_features] Coordinates of cluster centers. - labels_ : + labels_ : array of shape (n_samples,) Labels of each point. n_iter_ : int @@ -346,7 +346,8 @@ class MeanShift(ClusterMixin, BaseEstimator): Machine Intelligence. 2002. pp. 603-619. """ - def __init__(self, bandwidth=None, seeds=None, bin_seeding=False, + @_deprecate_positional_args + def __init__(self, *, bandwidth=None, seeds=None, bin_seeding=False, min_bin_freq=1, cluster_all=True, n_jobs=None, max_iter=300): self.bandwidth = bandwidth self.seeds = seeds @@ -367,7 +368,7 @@ def fit(self, X, y=None): y : Ignored """ - X = check_array(X) + X = self._validate_data(X) bandwidth = self.bandwidth if bandwidth is None: bandwidth = estimate_bandwidth(X, n_jobs=self.n_jobs) diff --git a/sklearn/cluster/_optics.py b/sklearn/cluster/_optics.py index ec2c45453d2be..92322b0ab0bfd 100755 --- a/sklearn/cluster/_optics.py +++ b/sklearn/cluster/_optics.py @@ -16,6 +16,7 @@ from ..utils import check_array from ..utils import gen_batches, get_chunk_n_rows +from ..utils.validation import _deprecate_positional_args from ..neighbors import NearestNeighbors from ..base import BaseEstimator, ClusterMixin from ..metrics import pairwise_distances @@ -203,10 +204,10 @@ class OPTICS(ClusterMixin, BaseEstimator): >>> clustering.labels_ array([0, 0, 0, 1, 1, 1]) """ - - def __init__(self, min_samples=5, max_eps=np.inf, metric='minkowski', p=2, - metric_params=None, cluster_method='xi', eps=None, xi=0.05, - predecessor_correction=True, min_cluster_size=None, + @_deprecate_positional_args + def __init__(self, *, min_samples=5, max_eps=np.inf, metric='minkowski', + p=2, metric_params=None, cluster_method='xi', eps=None, + xi=0.05, predecessor_correction=True, min_cluster_size=None, algorithm='auto', leaf_size=30, n_jobs=None): self.max_eps = max_eps self.min_samples = min_samples @@ -244,7 +245,7 @@ def fit(self, X, y=None): self : instance of OPTICS The instance. """ - X = check_array(X, dtype=np.float) + X = self._validate_data(X, dtype=np.float) if self.cluster_method not in ['dbscan', 'xi']: raise ValueError("cluster_method should be one of" diff --git a/sklearn/cluster/_spectral.py b/sklearn/cluster/_spectral.py index 78cdcc5073ccc..2faddabefa157 100644 --- a/sklearn/cluster/_spectral.py +++ b/sklearn/cluster/_spectral.py @@ -11,11 +11,11 @@ from ..base import BaseEstimator, ClusterMixin from ..utils import check_random_state, as_float_array -from ..utils.validation import check_array +from ..utils.validation import check_array, _deprecate_positional_args from ..metrics.pairwise import pairwise_kernels from ..neighbors import kneighbors_graph, NearestNeighbors from ..manifold import spectral_embedding -from ._k_means import k_means +from ._kmeans import k_means def discretize(vectors, copy=True, max_svd_restarts=30, n_iter_max=20, @@ -38,7 +38,7 @@ def discretize(vectors, copy=True, max_svd_restarts=30, n_iter_max=20, Maximum number of iterations to attempt in rotation and partition matrix search if machine precision convergence is not reached - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for rotation matrix initialization. Use an int to make the randomness deterministic. See :term:`Glossary `. @@ -194,7 +194,7 @@ def spectral_clustering(affinity, n_clusters=8, n_components=None, to be installed. It can be faster on very large, sparse problems, but may also lead to instabilities - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None A pseudo random number generator used for the initialization of the lobpcg eigen vectors decomposition when eigen_solver == 'amg' and by the K-Means initialization. Use an int to make the randomness @@ -310,7 +310,7 @@ class SpectralClustering(ClusterMixin, BaseEstimator): n_components : integer, optional, default=n_clusters Number of eigen vectors to use for the spectral embedding - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None A pseudo random number generator used for the initialization of the lobpcg eigen vectors decomposition when ``eigen_solver='amg'`` and by the K-Means initialization. Use an int to make the randomness @@ -433,8 +433,8 @@ class SpectralClustering(ClusterMixin, BaseEstimator): Stella X. Yu, Jianbo Shi https://www1.icsi.berkeley.edu/~stellayu/publication/doc/2003kwayICCV.pdf """ - - def __init__(self, n_clusters=8, eigen_solver=None, n_components=None, + @_deprecate_positional_args + def __init__(self, n_clusters=8, *, eigen_solver=None, n_components=None, random_state=None, n_init=10, gamma=1., affinity='rbf', n_neighbors=10, eigen_tol=0.0, assign_labels='kmeans', degree=3, coef0=1, kernel_params=None, n_jobs=None): @@ -474,8 +474,8 @@ def fit(self, X, y=None): self """ - X = check_array(X, accept_sparse=['csr', 'csc', 'coo'], - dtype=np.float64, ensure_min_samples=2) + X = self._validate_data(X, accept_sparse=['csr', 'csc', 'coo'], + dtype=np.float64, ensure_min_samples=2) allow_squared = self.affinity in ["precomputed", "precomputed_nearest_neighbors"] if X.shape[0] == X.shape[1] and not allow_squared: diff --git a/sklearn/cluster/setup.py b/sklearn/cluster/setup.py index a0ee8e62853c1..48ed25c5c0eaf 100644 --- a/sklearn/cluster/setup.py +++ b/sklearn/cluster/setup.py @@ -13,6 +13,7 @@ def configuration(parent_package='', top_path=None): libraries.append('m') config = Configuration('cluster', parent_package, top_path) + config.add_extension('_dbscan_inner', sources=['_dbscan_inner.pyx'], include_dirs=[numpy.get_include()], @@ -24,14 +25,19 @@ def configuration(parent_package='', top_path=None): include_dirs=[numpy.get_include()], libraries=libraries) - config.add_extension('_k_means_elkan', - sources=['_k_means_elkan.pyx'], + config.add_extension('_k_means_fast', + sources=['_k_means_fast.pyx'], include_dirs=[numpy.get_include()], libraries=libraries) - config.add_extension('_k_means_fast', - sources=['_k_means_fast.pyx'], - include_dirs=numpy.get_include(), + config.add_extension('_k_means_lloyd', + sources=['_k_means_lloyd.pyx'], + include_dirs=[numpy.get_include()], + libraries=libraries) + + config.add_extension('_k_means_elkan', + sources=['_k_means_elkan.pyx'], + include_dirs=[numpy.get_include()], libraries=libraries) config.add_subpackage('tests') diff --git a/sklearn/cluster/tests/test_bicluster.py b/sklearn/cluster/tests/test_bicluster.py index 38800de6a59cb..7d5a920600d7d 100644 --- a/sklearn/cluster/tests/test_bicluster.py +++ b/sklearn/cluster/tests/test_bicluster.py @@ -67,8 +67,7 @@ def test_spectral_coclustering(): 'n_svd_vecs': [None, 20], 'mini_batch': [False, True], 'init': ['k-means++'], - 'n_init': [10], - 'n_jobs': [1]} + 'n_init': [10]} random_state = 0 S, rows, cols = make_biclusters((30, 30), 3, noise=0.5, random_state=random_state) @@ -201,16 +200,13 @@ def test_project_and_cluster(): [0, 1], [0, 0]]) for mat in (data, csr_matrix(data)): - labels = model._project_and_cluster(data, vectors, + labels = model._project_and_cluster(mat, vectors, n_clusters=2) assert_almost_equal(v_measure_score(labels, [0, 0, 1, 1]), 1.0) def test_perfect_checkerboard(): - # XXX test always skipped - raise SkipTest("This test is failing on the buildbot, but cannot" - " reproduce. Temporarily disabling it until it can be" - " reproduced and fixed.") + # XXX Previously failed on build bot (not reproducible) model = SpectralBiclustering(3, svd_method="arpack", random_state=0) S, rows, cols = make_checkerboard((30, 30), 3, noise=0, @@ -256,3 +252,27 @@ def test_wrong_shape(): data = np.arange(27).reshape((3, 3, 3)) with pytest.raises(ValueError): model.fit(data) + + +@pytest.mark.parametrize('est', + (SpectralBiclustering(), SpectralCoclustering())) +def test_n_features_in_(est): + + X, _, _ = make_biclusters((3, 3), 3, random_state=0) + + assert not hasattr(est, 'n_features_in_') + est.fit(X) + assert est.n_features_in_ == 3 + + +@pytest.mark.parametrize("klass", [SpectralBiclustering, SpectralCoclustering]) +@pytest.mark.parametrize("n_jobs", [None, 1]) +def test_n_jobs_deprecated(klass, n_jobs): + # FIXME: remove in 0.25 + depr_msg = ("'n_jobs' was deprecated in version 0.23 and will be removed " + "in 0.25.") + S, _, _ = make_biclusters((30, 30), 3, noise=0.5, random_state=0) + est = klass(random_state=0, n_jobs=n_jobs) + + with pytest.warns(FutureWarning, match=depr_msg): + est.fit(S) diff --git a/sklearn/cluster/tests/test_birch.py b/sklearn/cluster/tests/test_birch.py index 06b74aaa9ba8d..3f815734e270a 100644 --- a/sklearn/cluster/tests/test_birch.py +++ b/sklearn/cluster/tests/test_birch.py @@ -159,3 +159,11 @@ def test_threshold(): brc = Birch(threshold=5.0, n_clusters=None) brc.fit(X) check_threshold(brc, 5.) + + +def test_birch_n_clusters_long_int(): + # Check that birch supports n_clusters with np.int64 dtype, for instance + # coming from np.arange. #16484 + X, _ = make_blobs(random_state=0) + n_clusters = np.int64(5) + Birch(n_clusters=n_clusters).fit(X) diff --git a/sklearn/cluster/tests/test_hierarchical.py b/sklearn/cluster/tests/test_hierarchical.py index bb93cb20395fd..91152ec7ca97a 100644 --- a/sklearn/cluster/tests/test_hierarchical.py +++ b/sklearn/cluster/tests/test_hierarchical.py @@ -22,8 +22,9 @@ from sklearn.cluster import ward_tree from sklearn.cluster import AgglomerativeClustering, FeatureAgglomeration -from sklearn.cluster._hierarchical import (_hc_cut, _TREE_BUILDERS, - linkage_tree, _fix_connectivity) +from sklearn.cluster._agglomerative import (_hc_cut, _TREE_BUILDERS, + linkage_tree, + _fix_connectivity) from sklearn.feature_extraction.image import grid_to_graph from sklearn.metrics.pairwise import PAIRED_DISTANCES, cosine_distances,\ manhattan_distances, pairwise_distances @@ -280,7 +281,7 @@ def assess_same_labelling(cut1, cut2): assert (co_clust[0] == co_clust[1]).all() -def test_scikit_vs_scipy(): +def test_sparse_scikit_vs_scipy(): # Test scikit linkage with full connectivity (i.e. unstructured) vs scipy n, p, k = 10, 5, 3 rng = np.random.RandomState(0) @@ -314,6 +315,33 @@ def test_scikit_vs_scipy(): _hc_cut(n_leaves + 1, children, n_leaves) +# Make sure our custom mst_linkage_core gives +# the same results as scipy's builtin +@pytest.mark.parametrize('seed', range(5)) +def test_vector_scikit_single_vs_scipy_single(seed): + n_samples, n_features, n_clusters = 10, 5, 3 + rng = np.random.RandomState(seed) + X = .1 * rng.normal(size=(n_samples, n_features)) + X -= 4. * np.arange(n_samples)[:, np.newaxis] + X -= X.mean(axis=1)[:, np.newaxis] + + out = hierarchy.linkage(X, method='single') + children_scipy = out[:, :2].astype(np.int) + + children, _, n_leaves, _ = _TREE_BUILDERS['single'](X) + + # Sort the order of child nodes per row for consistency + children.sort(axis=1) + assert_array_equal(children, children_scipy, + 'linkage tree differs' + ' from scipy impl for' + ' single linkage.') + + cut = _hc_cut(n_clusters, children, n_leaves) + cut_scipy = _hc_cut(n_clusters, children_scipy, n_leaves) + assess_same_labelling(cut, cut_scipy) + + def test_identical_points(): # Ensure identical points are handled correctly when using mst with # a sparse connectivity matrix @@ -725,15 +753,11 @@ def test_dist_threshold_invalid_parameters(): compute_full_tree=False).fit(X) -def test_n_components_deprecation(): - # Test that a Deprecation warning is thrown when n_components_ - # attribute is accessed - - X = np.array([[1, 2], [1, 4], [1, 0], [4, 2]]) - agc = AgglomerativeClustering().fit(X) - - match = ("``n_components_`` attribute was deprecated " - "in favor of ``n_connected_components_``") - with pytest.warns(FutureWarning, match=match): - n = agc.n_components_ - assert n == agc.n_connected_components_ +def test_invalid_shape_precomputed_dist_matrix(): + # Check that an error is raised when affinity='precomputed' + # and a non square matrix is passed (PR #16257). + rng = np.random.RandomState(0) + X = rng.rand(5, 3) + with pytest.raises(ValueError, match="Distance matrix should be square, "): + AgglomerativeClustering(affinity='precomputed', + linkage='complete').fit(X) diff --git a/sklearn/cluster/tests/test_k_means.py b/sklearn/cluster/tests/test_k_means.py index ea7e5b7825437..2bcbc3faa517f 100644 --- a/sklearn/cluster/tests/test_k_means.py +++ b/sklearn/cluster/tests/test_k_means.py @@ -3,6 +3,7 @@ import numpy as np from scipy import sparse as sp +from threadpoolctl import threadpool_limits import pytest @@ -12,18 +13,24 @@ from sklearn.utils._testing import assert_almost_equal from sklearn.utils._testing import assert_warns from sklearn.utils._testing import assert_warns_message -from sklearn.utils._testing import if_safe_multiprocessing_with_blas from sklearn.utils._testing import assert_raise_message from sklearn.utils.validation import _num_samples from sklearn.base import clone from sklearn.exceptions import ConvergenceWarning from sklearn.utils.extmath import row_norms +from sklearn.metrics import pairwise_distances_argmin from sklearn.metrics.cluster import v_measure_score from sklearn.cluster import KMeans, k_means from sklearn.cluster import MiniBatchKMeans -from sklearn.cluster._k_means import _labels_inertia -from sklearn.cluster._k_means import _mini_batch_step +from sklearn.cluster._kmeans import _labels_inertia +from sklearn.cluster._kmeans import _mini_batch_step +from sklearn.cluster._k_means_fast import _relocate_empty_clusters_dense +from sklearn.cluster._k_means_fast import _relocate_empty_clusters_sparse +from sklearn.cluster._k_means_fast import _euclidean_dense_dense_wrapper +from sklearn.cluster._k_means_fast import _euclidean_sparse_dense_wrapper +from sklearn.cluster._k_means_fast import _inertia_dense +from sklearn.cluster._k_means_fast import _inertia_sparse from sklearn.datasets import make_blobs from io import StringIO from sklearn.metrics.cluster import homogeneity_score @@ -42,10 +49,8 @@ X_csr = sp.csr_matrix(X) -@pytest.mark.parametrize("representation, algo", - [('dense', 'full'), - ('dense', 'elkan'), - ('sparse', 'full')]) +@pytest.mark.parametrize("representation", ["dense", "sparse"]) +@pytest.mark.parametrize("algo", ["full", "elkan"]) @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_kmeans_results(representation, algo, dtype): # cheks that kmeans works as intended @@ -68,24 +73,124 @@ def test_kmeans_results(representation, algo, dtype): assert kmeans.n_iter_ == expected_n_iter +@pytest.mark.parametrize("array_constr", + [np.array, sp.csr_matrix], + ids=['dense', 'sparse']) +@pytest.mark.parametrize("algo", ['full', 'elkan']) +def test_relocated_clusters(array_constr, algo): + # check that empty clusters are relocated as expected + X = array_constr([[0, 0], [0.5, 0], [0.5, 1], [1, 1]]) + + # second center too far from others points will be empty at first iter + init_centers = np.array([[0.5, 0.5], [3, 3]]) + + expected_labels = [0, 0, 1, 1] + expected_inertia = 0.25 + expected_centers = [[0.25, 0], [0.75, 1]] + expected_n_iter = 3 + + kmeans = KMeans(n_clusters=2, n_init=1, init=init_centers, algorithm=algo) + kmeans.fit(X) + + assert_array_equal(kmeans.labels_, expected_labels) + assert_almost_equal(kmeans.inertia_, expected_inertia) + assert_array_almost_equal(kmeans.cluster_centers_, expected_centers) + assert kmeans.n_iter_ == expected_n_iter + + +@pytest.mark.parametrize("representation", ["dense", "sparse"]) +def test_relocate_empty_clusters(representation): + # test for the _relocate_empty_clusters_(dense/sparse) helpers + + # Synthetic dataset with 3 obvious clusters of different sizes + X = np.array( + [-10., -9.5, -9, -8.5, -8, -1, 1, 9, 9.5, 10]).reshape(-1, 1) + if representation == "sparse": + X = sp.csr_matrix(X) + sample_weight = np.full(shape=10, fill_value=1.) + + # centers all initialized to the first point of X + centers_old = np.array([-10., -10, -10]).reshape(-1, 1) + + # With this initialization, all points will be assigned to the first center + # At this point a center in centers_new is the weighted sum of the points + # it contains if it's not empty, otherwise it is the same as before. + centers_new = np.array([-16.5, -10, -10]).reshape(-1, 1) + weight_in_clusters = np.array([10., 0, 0]) + labels = np.zeros(10, dtype=np.int32) + + if representation == "dense": + _relocate_empty_clusters_dense(X, sample_weight, centers_old, + centers_new, weight_in_clusters, labels) + else: + _relocate_empty_clusters_sparse(X.data, X.indices, X.indptr, + sample_weight, centers_old, + centers_new, weight_in_clusters, + labels) + + # The relocation scheme will take the 2 points farthest from the center and + # assign them to the 2 empty clusters, i.e. points at 10 and at 9.9. The + # first center will be updated to contain the other 8 points. + assert_array_equal(weight_in_clusters, [8, 1, 1]) + assert_allclose(centers_new, [[-36], [10], [9.5]]) + + @pytest.mark.parametrize('distribution', ['normal', 'blobs']) -def test_elkan_results(distribution): +@pytest.mark.parametrize('tol', [1e-2, 1e-4, 1e-8]) +def test_elkan_results(distribution, tol): # check that results are identical between lloyd and elkan algorithms rnd = np.random.RandomState(0) if distribution == 'normal': - X = rnd.normal(size=(50, 10)) + X = rnd.normal(size=(5000, 10)) else: X, _ = make_blobs(random_state=rnd) + km_full = KMeans(algorithm='full', n_clusters=5, + random_state=0, n_init=1, tol=tol) + km_elkan = KMeans(algorithm='elkan', n_clusters=5, + random_state=0, n_init=1, tol=tol) + + km_full.fit(X) + km_elkan.fit(X) + assert_allclose(km_elkan.cluster_centers_, km_full.cluster_centers_) + assert_array_equal(km_elkan.labels_, km_full.labels_) + + assert km_elkan.n_iter_ == km_full.n_iter_ + assert km_elkan.inertia_ == pytest.approx(km_full.inertia_, rel=1e-6) + + +@pytest.mark.parametrize('algorithm', ['full', 'elkan']) +def test_kmeans_convergence(algorithm): + # Check that KMeans stops when convergence is reached when tol=0. (#16075) + rnd = np.random.RandomState(0) + X = rnd.normal(size=(5000, 10)) + + km = KMeans(algorithm=algorithm, n_clusters=5, random_state=0, n_init=1, + tol=0, max_iter=300).fit(X) + + assert km.n_iter_ < 300 + + +@pytest.mark.parametrize('distribution', ['normal', 'blobs']) +def test_elkan_results_sparse(distribution): + # check that results are identical between lloyd and elkan algorithms + # with sparse input + rnd = np.random.RandomState(0) + if distribution == 'normal': + X = sp.random(100, 100, density=0.1, format='csr', random_state=rnd) + X.data = rnd.randn(len(X.data)) + else: + X, _ = make_blobs(n_samples=100, n_features=100, random_state=rnd) + X = sp.csr_matrix(X) + km_full = KMeans(algorithm='full', n_clusters=5, random_state=0, n_init=1) km_elkan = KMeans(algorithm='elkan', n_clusters=5, random_state=0, n_init=1) km_full.fit(X) km_elkan.fit(X) - assert_array_almost_equal(km_elkan.cluster_centers_, - km_full.cluster_centers_) - assert_array_equal(km_elkan.labels_, km_full.labels_) + assert_allclose(km_elkan.cluster_centers_, km_full.cluster_centers_) + assert_allclose(km_elkan.labels_, km_full.labels_) def test_labels_assignment_and_inertia(): @@ -230,33 +335,6 @@ def test_k_means_new_centers(): np.testing.assert_array_equal(this_labels, labels) -@if_safe_multiprocessing_with_blas -def test_k_means_plus_plus_init_2_jobs(): - km = KMeans(init="k-means++", n_clusters=n_clusters, n_jobs=2, - random_state=42).fit(X) - _check_fitted_model(km) - - -def test_k_means_precompute_distances_flag(): - # check that a warning is raised if the precompute_distances flag is not - # supported - km = KMeans(precompute_distances="wrong") - with pytest.raises(ValueError): - km.fit(X) - - -def test_k_means_plus_plus_init_not_precomputed(): - km = KMeans(init="k-means++", n_clusters=n_clusters, random_state=42, - precompute_distances=False).fit(X) - _check_fitted_model(km) - - -def test_k_means_random_init_not_precomputed(): - km = KMeans(init="random", n_clusters=n_clusters, random_state=42, - precompute_distances=False).fit(X) - _check_fitted_model(km) - - @pytest.mark.parametrize('data', [X, X_csr], ids=['dense', 'sparse']) @pytest.mark.parametrize('init', ['random', 'k-means++', centers.copy()]) def test_k_means_init(data, init): @@ -313,8 +391,7 @@ def test_k_means_fortran_aligned_data(): X = np.asfortranarray([[0, 0], [0, 1], [0, 1]]) centers = np.array([[0, 0], [0, 1]]) labels = np.array([0, 1, 1]) - km = KMeans(n_init=1, init=centers, precompute_distances=False, - random_state=42, n_clusters=2) + km = KMeans(n_init=1, init=centers, random_state=42, n_clusters=2) km.fit(X) assert_array_almost_equal(km.cluster_centers_, centers) assert_array_equal(km.labels_, labels) @@ -342,20 +419,24 @@ def test_k_means_fit_predict(algo, dtype, constructor, seed, max_iter, tol): pytest.xfail( "Known failures on MacOS, See " "https://github.com/scikit-learn/scikit-learn/issues/12644") - if not (algo == 'elkan' and constructor is sp.csr_matrix): - rng = np.random.RandomState(seed) - X = make_blobs(n_samples=1000, n_features=10, centers=10, - random_state=rng)[0].astype(dtype, copy=False) - X = constructor(X) + rng = np.random.RandomState(seed) - kmeans = KMeans(algorithm=algo, n_clusters=10, random_state=seed, - tol=tol, max_iter=max_iter, n_jobs=1) + X = make_blobs(n_samples=1000, n_features=10, centers=10, + random_state=rng)[0].astype(dtype, copy=False) + X = constructor(X) - labels_1 = kmeans.fit(X).predict(X) - labels_2 = kmeans.fit_predict(X) + kmeans = KMeans(algorithm=algo, n_clusters=10, random_state=seed, + tol=tol, max_iter=max_iter) - assert_array_equal(labels_1, labels_2) + labels_1 = kmeans.fit(X).predict(X) + labels_2 = kmeans.fit_predict(X) + + # Due to randomness in the order in which chunks of data are processed when + # using more than one thread, the absolute values of the labels can be + # different between the 2 strategies but they should correspond to the same + # clustering. + assert v_measure_score(labels_1, labels_2) == 1 def test_mb_kmeans_verbose(): @@ -666,7 +747,7 @@ def test_fit_transform(): @pytest.mark.parametrize('algo', ['full', 'elkan']) def test_predict_equal_labels(algo): - km = KMeans(random_state=13, n_jobs=1, n_init=1, max_iter=1, + km = KMeans(random_state=13, n_init=1, max_iter=1, algorithm=algo) km.fit(X) assert_array_equal(km.predict(X), km.labels_) @@ -726,15 +807,10 @@ def test_k_means_function(): with pytest.raises(ValueError): k_means(X, n_clusters=X.shape[0] + 1, sample_weight=None) - # kmeans for algorithm='elkan' raises TypeError on sparse matrix - assert_raise_message(TypeError, "algorithm='elkan' not supported for " - "sparse input X", k_means, X=X_csr, n_clusters=2, - sample_weight=None, algorithm="elkan") - def test_x_squared_norms_init_centroids(): # Test that x_squared_norms can be None in _init_centroids - from sklearn.cluster._k_means import _init_centroids + from sklearn.cluster._kmeans import _init_centroids X_norms = np.sum(X**2, axis=1) precompute = _init_centroids( @@ -773,8 +849,7 @@ def test_float_precision(Estimator, is_sparse): X_new[dtype] = estimator.transform(X_test) centers[dtype] = estimator.cluster_centers_ # ensure the extracted row is a 2d array - assert (estimator.predict(X_test[:1]) == - estimator.labels_[0]) + assert estimator.predict(X_test[:1]) == estimator.labels_[0] if hasattr(estimator, 'partial_fit'): estimator.partial_fit(X_test[0:3]) # dtype of cluster centers has to stay the same after @@ -921,7 +996,7 @@ def test_sample_weight_length(): def test_check_normalize_sample_weight(): - from sklearn.cluster._k_means import _check_normalize_sample_weight + from sklearn.cluster._kmeans import _check_normalize_sample_weight sample_weight = None checked_sample_weight = _check_normalize_sample_weight(sample_weight, X) assert _num_samples(X) == _num_samples(checked_sample_weight) @@ -959,11 +1034,136 @@ def test_minibatch_kmeans_partial_fit_int_data(): assert km.cluster_centers_.dtype.kind == "f" -def test_result_of_kmeans_equal_in_diff_n_jobs(): - # PR 9288 +def test_result_of_kmeans_equal_in_diff_n_threads(): + # Check that KMeans gives the same results in parallel mode than in + # sequential mode. rnd = np.random.RandomState(0) X = rnd.normal(size=(50, 10)) - result_1 = KMeans(n_clusters=3, random_state=0, n_jobs=1).fit(X).labels_ - result_2 = KMeans(n_clusters=3, random_state=0, n_jobs=2).fit(X).labels_ + with threadpool_limits(limits=1, user_api="openmp"): + result_1 = KMeans( + n_clusters=3, random_state=0).fit(X).labels_ + with threadpool_limits(limits=2, user_api="openmp"): + result_2 = KMeans( + n_clusters=3, random_state=0).fit(X).labels_ assert_array_equal(result_1, result_2) + + +@pytest.mark.parametrize("precompute_distances", ["auto", False, True]) +def test_precompute_distance_deprecated(precompute_distances): + # FIXME: remove in 0.25 + depr_msg = ("'precompute_distances' was deprecated in version 0.23 and " + "will be removed in 0.25.") + X, _ = make_blobs(n_samples=10, n_features=2, centers=2, random_state=0) + kmeans = KMeans(n_clusters=2, n_init=1, init='random', random_state=0, + precompute_distances=precompute_distances) + + with pytest.warns(FutureWarning, match=depr_msg): + kmeans.fit(X) + + +@pytest.mark.parametrize("n_jobs", [None, 1]) +def test_n_jobs_deprecated(n_jobs): + # FIXME: remove in 0.25 + depr_msg = ("'n_jobs' was deprecated in version 0.23 and will be removed " + "in 0.25.") + X, _ = make_blobs(n_samples=10, n_features=2, centers=2, random_state=0) + kmeans = KMeans(n_clusters=2, n_init=1, init='random', random_state=0, + n_jobs=n_jobs) + + with pytest.warns(FutureWarning, match=depr_msg): + kmeans.fit(X) + + +def test_warning_elkan_1_cluster(): + X, _ = make_blobs(n_samples=10, n_features=2, centers=1, random_state=0) + kmeans = KMeans(n_clusters=1, n_init=1, init='random', random_state=0, + algorithm='elkan') + + with pytest.warns(RuntimeWarning, + match="algorithm='elkan' doesn't make sense for a single" + " cluster"): + kmeans.fit(X) + + +def test_error_wrong_algorithm(): + X, _ = make_blobs(n_samples=10, n_features=2, centers=2, random_state=0) + kmeans = KMeans(n_clusters=2, n_init=1, init='random', random_state=0, + algorithm='wrong') + + with pytest.raises(ValueError, + match="Algorithm must be 'auto', 'full' or 'elkan'"): + kmeans.fit(X) + + +@pytest.mark.parametrize("array_constr", + [np.array, sp.csr_matrix], + ids=['dense', 'sparse']) +@pytest.mark.parametrize("algo", ['full', 'elkan']) +def test_k_means_1_iteration(array_constr, algo): + # check the results after a single iteration (E-step M-step E-step) by + # comparing against a pure python implementation. + X = np.random.RandomState(0).uniform(size=(100, 5)) + init_centers = X[:5] + X = array_constr(X) + + def py_kmeans(X, init): + new_centers = init.copy() + labels = pairwise_distances_argmin(X, init) + for label in range(init.shape[0]): + new_centers[label] = X[labels == label].mean(axis=0) + labels = pairwise_distances_argmin(X, new_centers) + return labels, new_centers + + py_labels, py_centers = py_kmeans(X, init_centers) + + cy_kmeans = KMeans(n_clusters=5, n_init=1, init=init_centers, + algorithm=algo, max_iter=1).fit(X) + cy_labels = cy_kmeans.labels_ + cy_centers = cy_kmeans.cluster_centers_ + + assert_array_equal(py_labels, cy_labels) + assert_allclose(py_centers, cy_centers) + + +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +@pytest.mark.parametrize("squared", [True, False]) +def test_euclidean_distance(dtype, squared): + rng = np.random.RandomState(0) + a_sparse = sp.random(1, 100, density=0.5, format="csr", random_state=rng, + dtype=dtype) + a_dense = a_sparse.toarray().reshape(-1) + b = rng.randn(100).astype(dtype, copy=False) + b_squared_norm = (b**2).sum() + + expected = ((a_dense - b)**2).sum() + expected = expected if squared else np.sqrt(expected) + + distance_dense_dense = _euclidean_dense_dense_wrapper(a_dense, b, squared) + distance_sparse_dense = _euclidean_sparse_dense_wrapper( + a_sparse.data, a_sparse.indices, b, b_squared_norm, squared) + + assert_allclose(distance_dense_dense, distance_sparse_dense, rtol=1e-6) + assert_allclose(distance_dense_dense, expected, rtol=1e-6) + assert_allclose(distance_sparse_dense, expected, rtol=1e-6) + + +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +def test_inertia(dtype): + rng = np.random.RandomState(0) + X_sparse = sp.random(100, 10, density=0.5, format="csr", random_state=rng, + dtype=dtype) + X_dense = X_sparse.toarray() + sample_weight = rng.randn(100).astype(dtype, copy=False) + centers = rng.randn(5, 10).astype(dtype, copy=False) + labels = rng.randint(5, size=100, dtype=np.int32) + + distances = ((X_dense - centers[labels])**2).sum(axis=1) + expected = np.sum(distances * sample_weight) + + inertia_dense = _inertia_dense(X_dense, sample_weight, centers, labels) + inertia_sparse = _inertia_sparse(X_sparse, sample_weight, centers, labels) + + assert_allclose(inertia_dense, inertia_sparse, rtol=1e-6) + assert_allclose(inertia_dense, expected, rtol=1e-6) + assert_allclose(inertia_sparse, expected, rtol=1e-6) diff --git a/sklearn/cluster/tests/test_spectral.py b/sklearn/cluster/tests/test_spectral.py index dc79f427afcdf..f5591c7348ebe 100644 --- a/sklearn/cluster/tests/test_spectral.py +++ b/sklearn/cluster/tests/test_spectral.py @@ -191,6 +191,10 @@ def test_discretize(n_samples): assert adjusted_rand_score(y_true, y_pred) > 0.8 +# TODO: Remove when pyamg does replaces sp.rand call with np.random.rand +# https://github.com/scikit-learn/scikit-learn/issues/15913 +@pytest.mark.filterwarnings( + "ignore:scipy.rand is deprecated:DeprecationWarning:pyamg.*") def test_spectral_clustering_with_arpack_amg_solvers(): # Test that spectral_clustering is the same for arpack and amg solver # Based on toy example from plot_segmentation_toy.py diff --git a/sklearn/compose/_column_transformer.py b/sklearn/compose/_column_transformer.py index 0b6a7363686a9..e94757bca6993 100644 --- a/sklearn/compose/_column_transformer.py +++ b/sklearn/compose/_column_transformer.py @@ -51,20 +51,20 @@ class ColumnTransformer(TransformerMixin, _BaseComposition): Parameters ---------- transformers : list of tuples - List of (name, transformer, column(s)) tuples specifying the + List of (name, transformer, columns) tuples specifying the transformer objects to be applied to subsets of the data. - name : string + name : str Like in Pipeline and FeatureUnion, this allows the transformer and its parameters to be set using ``set_params`` and searched in grid search. - transformer : estimator or {'passthrough', 'drop'} + transformer : {'drop', 'passthrough'} or estimator Estimator must support :term:`fit` and :term:`transform`. Special-cased strings 'drop' and 'passthrough' are accepted as well, to indicate to drop the columns or to pass them through untransformed, respectively. - column(s) : string or int, array-like of string or int, slice, \ -boolean mask array or callable + columns : str, array-like of str, int, array-like of int, \ + array-like of bool, slice or callable Indexes the data on its second axis. Integers are interpreted as positional columns, while strings can reference DataFrame columns by name. A scalar string or int should be used where @@ -72,9 +72,9 @@ class ColumnTransformer(TransformerMixin, _BaseComposition): otherwise a 2d array will be passed to the transformer. A callable is passed the input data `X` and can return any of the above. To select multiple columns by name or dtype, you can use - :obj:`make_column_transformer`. + :obj:`make_column_selector`. - remainder : {'drop', 'passthrough'} or estimator, default 'drop' + remainder : {'drop', 'passthrough'} or estimator, default='drop' By default, only the specified columns in `transformers` are transformed and combined in the output, and the non-specified columns are dropped. (default of ``'drop'``). @@ -88,25 +88,25 @@ class ColumnTransformer(TransformerMixin, _BaseComposition): Note that using this feature requires that the DataFrame columns input at :term:`fit` and :term:`transform` have identical order. - sparse_threshold : float, default = 0.3 + sparse_threshold : float, default=0.3 If the output of the different transformers contains sparse matrices, these will be stacked as a sparse matrix if the overall density is lower than this value. Use ``sparse_threshold=0`` to always return dense. When the transformed output consists of all dense data, the stacked result will be dense, and this keyword will be ignored. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None Number of jobs to run in parallel. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - transformer_weights : dict, optional + transformer_weights : dict, default=None Multiplicative weights for features per transformer. The output of the transformer is multiplied by these weights. Keys are transformer names, values the weights. - verbose : boolean, optional(default=False) + verbose : bool, default=False If True, the time elapsed while fitting each transformer will be printed as it is completed. @@ -124,13 +124,13 @@ class ColumnTransformer(TransformerMixin, _BaseComposition): ``len(transformers_)==len(transformers)+1``, otherwise ``len(transformers_)==len(transformers)``. - named_transformers_ : Bunch object, a dictionary with attribute access + named_transformers_ : :class:`~sklearn.utils.Bunch` Read-only attribute to access any transformer by given name. Keys are transformer names and values are the fitted transformer objects. - sparse_output_ : boolean - Boolean flag indicating wether the output of ``transform`` is a + sparse_output_ : bool + Boolean flag indicating whether the output of ``transform`` is a sparse matrix or a dense numpy array, which depends on the output of the individual transformers and the `sparse_threshold` keyword. @@ -206,13 +206,13 @@ def get_params(self, deep=True): Parameters ---------- - deep : boolean, optional + deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns ------- - params : mapping of string to any + params : dict Parameter names mapped to their values. """ return self._get_params('_transformers', deep=deep) @@ -350,8 +350,9 @@ def get_feature_names(self): """ check_is_fitted(self) feature_names = [] - for name, trans, _, _ in self._iter(fitted=True): - if trans == 'drop': + for name, trans, column, _ in self._iter(fitted=True): + if trans == 'drop' or ( + hasattr(column, '__len__') and not len(column)): continue elif trans == 'passthrough': raise NotImplementedError( @@ -466,11 +467,11 @@ def fit(self, X, y=None): Parameters ---------- - X : array-like or DataFrame of shape [n_samples, n_features] + X : {array-like, dataframe} of shape (n_samples, n_features) Input data, of which specified subsets are used to fit the transformers. - y : array-like, shape (n_samples, ...), optional + y : array-like of shape (n_samples,...), default=None Targets for supervised learning. Returns @@ -489,16 +490,17 @@ def fit_transform(self, X, y=None): Parameters ---------- - X : array-like or DataFrame of shape [n_samples, n_features] + X : {array-like, dataframe} of shape (n_samples, n_features) Input data, of which specified subsets are used to fit the transformers. - y : array-like, shape (n_samples, ...), optional + y : array-like of shape (n_samples,), default=None Targets for supervised learning. Returns ------- - X_t : array-like or sparse matrix, shape (n_samples, sum_n_components) + X_t : {array-like, sparse matrix} of \ + shape (n_samples, sum_n_components) hstack of results of transformers. sum_n_components is the sum of n_components (output dimension) over transformers. If any result is a sparse matrix, everything will be converted to @@ -511,6 +513,8 @@ def fit_transform(self, X, y=None): else: self._feature_names_in = None X = _check_X(X) + # set n_features_in_ attribute + self._check_n_features(X, reset=True) self._validate_transformers() self._validate_column_callables(X) self._validate_remainder(X) @@ -544,12 +548,13 @@ def transform(self, X): Parameters ---------- - X : array-like or DataFrame of shape [n_samples, n_features] + X : {array-like, dataframe} of shape (n_samples, n_features) The data to be transformed by subset. Returns ------- - X_t : array-like or sparse matrix, shape (n_samples, sum_n_components) + X_t : {array-like, sparse matrix} of \ + shape (n_samples, sum_n_components) hstack of results of transformers. sum_n_components is the sum of n_components (output dimension) over transformers. If any result is a sparse matrix, everything will be converted to @@ -584,6 +589,7 @@ def transform(self, X): 'and for transform when using the ' 'remainder keyword') + # TODO: also call _check_n_features(reset=False) in 0.24 self._validate_features(X.shape[1], X_feature_names) Xs = self._fit_transform(X, None, _transform_one, fitted=True) self._validate_output(Xs) @@ -602,7 +608,7 @@ def _hstack(self, Xs): Parameters ---------- - Xs : List of numpy arrays, sparse arrays, or DataFrames + Xs : list of {array-like, sparse matrix, dataframe} """ if self.sparse_output_: try: @@ -669,25 +675,26 @@ def make_column_transformer(*transformers, **kwargs): Parameters ---------- *transformers : tuples - Tuples of the form (transformer, column(s)) specifying the + Tuples of the form (transformer, columns) specifying the transformer objects to be applied to subsets of the data. - transformer : estimator or {'passthrough', 'drop'} + transformer : {'drop', 'passthrough'} or estimator Estimator must support :term:`fit` and :term:`transform`. Special-cased strings 'drop' and 'passthrough' are accepted as well, to indicate to drop the columns or to pass them through untransformed, respectively. - column(s) : string or int, array-like of string or int, slice, \ -boolean mask array or callable + columns : str, array-like of str, int, array-like of int, slice, \ + array-like of bool or callable Indexes the data on its second axis. Integers are interpreted as positional columns, while strings can reference DataFrame columns by name. A scalar string or int should be used where ``transformer`` expects X to be a 1d array-like (vector), otherwise a 2d array will be passed to the transformer. A callable is passed the input data `X` and can return any of the - above. + above. To select multiple columns by name or dtype, you can use + :obj:`make_column_selector`. - remainder : {'drop', 'passthrough'} or estimator, default 'drop' + remainder : {'drop', 'passthrough'} or estimator, default='drop' By default, only the specified columns in `transformers` are transformed and combined in the output, and the non-specified columns are dropped. (default of ``'drop'``). @@ -699,7 +706,7 @@ def make_column_transformer(*transformers, **kwargs): non-specified columns will use the ``remainder`` estimator. The estimator must support :term:`fit` and :term:`transform`. - sparse_threshold : float, default = 0.3 + sparse_threshold : float, default=0.3 If the transformed output consists of a mix of sparse and dense data, it will be stacked as a sparse matrix if the density is lower than this value. Use ``sparse_threshold=0`` to always return dense. @@ -707,13 +714,13 @@ def make_column_transformer(*transformers, **kwargs): the stacked result will be sparse or dense, respectively, and this keyword will be ignored. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None Number of jobs to run in parallel. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - verbose : boolean, optional(default=False) + verbose : bool, default=False If True, the time elapsed while fitting each transformer will be printed as it is completed. diff --git a/sklearn/compose/_target.py b/sklearn/compose/_target.py index 50a44cdb42b9a..27f4ef63edf68 100644 --- a/sklearn/compose/_target.py +++ b/sklearn/compose/_target.py @@ -10,6 +10,7 @@ from ..utils.validation import check_is_fitted from ..utils import check_array, _safe_indexing from ..preprocessing import FunctionTransformer +from ..exceptions import NotFittedError __all__ = ['TransformedTargetRegressor'] @@ -42,9 +43,10 @@ class TransformedTargetRegressor(RegressorMixin, BaseEstimator): Parameters ---------- - regressor : object, default=LinearRegression() + regressor : object, default=None Regressor object such as derived from ``RegressorMixin``. This regressor will automatically be cloned each time prior to fitting. + If regressor is ``None``, ``LinearRegression()`` is created and used. transformer : object, default=None Estimator object such as derived from ``TransformerMixin``. Cannot be @@ -54,13 +56,13 @@ class TransformedTargetRegressor(RegressorMixin, BaseEstimator): transformer will be cloned during fitting. Also, the transformer is restricting ``y`` to be a numpy array. - func : function, optional + func : function, default=None Function to apply to ``y`` before passing to ``fit``. Cannot be set at the same time as ``transformer``. The function needs to return a 2-dimensional array. If ``func`` is ``None``, the function used will be the identity function. - inverse_func : function, optional + inverse_func : function, default=None Function to apply to the prediction of the regressor. Cannot be set at the same time as ``transformer`` as well. The function needs to return a 2-dimensional array. The inverse function is used to return @@ -153,14 +155,14 @@ def fit(self, X, y, **fit_params): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training vector, where n_samples is the number of samples and n_features is the number of features. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) Target values. - **fit_params : dict of string -> object + **fit_params : dict Parameters passed to the ``fit`` method of the underlying regressor. @@ -215,7 +217,7 @@ def predict(self, X): Returns ------- - y_hat : array, shape = (n_samples,) + y_hat : ndarray of shape (n_samples,) Predicted values. """ @@ -234,3 +236,17 @@ def predict(self, X): def _more_tags(self): return {'poor_score': True, 'no_validation': True} + + @property + def n_features_in_(self): + # For consistency with other estimators we raise a AttributeError so + # that hasattr() returns False the estimator isn't fitted. + try: + check_is_fitted(self) + except NotFittedError as nfe: + raise AttributeError( + "{} object has no n_features_in_ attribute." + .format(self.__class__.__name__) + ) from nfe + + return self.regressor_.n_features_in_ diff --git a/sklearn/compose/tests/test_column_transformer.py b/sklearn/compose/tests/test_column_transformer.py index b635842e3d648..ca1c185c91e06 100644 --- a/sklearn/compose/tests/test_column_transformer.py +++ b/sklearn/compose/tests/test_column_transformer.py @@ -523,7 +523,8 @@ def predict(self, X): X_array = np.array([[0, 1, 2], [2, 4, 6]]).T ct = ColumnTransformer([('trans', NoTrans(), [0])]) - assert_raise_message(TypeError, "All estimators should implement fit", + assert_raise_message(TypeError, + "All estimators should implement fit and transform", ct.fit, X_array) @@ -1039,7 +1040,7 @@ def test_column_transformer_no_estimators_set_params(): def test_column_transformer_callable_specifier(): - # assert that function gets the full array / dataframe + # assert that function gets the full array X_array = np.array([[0, 1, 2], [2, 4, 6]]).T X_res_first = np.array([[0, 1, 2]]).T @@ -1054,7 +1055,13 @@ def func(X): assert callable(ct.transformers[0][2]) assert ct.transformers_[0][2] == [0] + +def test_column_transformer_callable_specifier_dataframe(): + # assert that function gets the full dataframe pd = pytest.importorskip('pandas') + X_array = np.array([[0, 1, 2], [2, 4, 6]]).T + X_res_first = np.array([[0, 1, 2]]).T + X_df = pd.DataFrame(X_array, columns=['first', 'second']) def func(X): @@ -1186,6 +1193,18 @@ def test_column_transformer_mask_indexing(array_type): assert X_trans.shape == (3, 2) +def test_n_features_in(): + # make sure n_features_in is what is passed as input to the column + # transformer. + + X = [[1, 2], [3, 4], [5, 6]] + ct = ColumnTransformer([('a', DoubleTrans(), [0]), + ('b', DoubleTrans(), [1])]) + assert not hasattr(ct, 'n_features_in_') + ct.fit(X) + assert ct.n_features_in_ == 2 + + @pytest.mark.parametrize('cols, pattern, include, exclude', [ (['col_int', 'col_float'], None, np.number, None), (['col_int', 'col_float'], None, None, object), @@ -1266,3 +1285,23 @@ def test_make_column_selector_pickle(): selector_picked = pickle.loads(pickle.dumps(selector)) assert_array_equal(selector(X_df), selector_picked(X_df)) + + +@pytest.mark.parametrize( + 'empty_col', [[], np.array([], dtype=np.int), lambda x: []], + ids=['list', 'array', 'callable'] +) +def test_feature_names_empty_columns(empty_col): + pd = pytest.importorskip('pandas') + + df = pd.DataFrame({"col1": ["a", "a", "b"], "col2": ["z", "z", "z"]}) + + ct = ColumnTransformer( + transformers=[ + ("ohe", OneHotEncoder(), ["col1", "col2"]), + ("empty_features", OneHotEncoder(), empty_col), + ], + ) + + ct.fit(df) + assert ct.get_feature_names() == ['ohe__x0_a', 'ohe__x0_b', 'ohe__x1_z'] diff --git a/sklearn/compose/tests/test_target.py b/sklearn/compose/tests/test_target.py index 4ba598b4c10ea..573518b3fa43a 100644 --- a/sklearn/compose/tests/test_target.py +++ b/sklearn/compose/tests/test_target.py @@ -16,7 +16,7 @@ from sklearn.pipeline import Pipeline -from sklearn.linear_model import LinearRegression, Lasso +from sklearn.linear_model import LinearRegression, OrthogonalMatchingPursuit from sklearn import datasets @@ -37,7 +37,7 @@ def test_transform_target_regressor_error(): regr.fit(X, y) # fit with sample_weight with a regressor which does not support it sample_weight = np.ones((y.shape[0],)) - regr = TransformedTargetRegressor(regressor=Lasso(), + regr = TransformedTargetRegressor(regressor=OrthogonalMatchingPursuit(), transformer=StandardScaler()) with pytest.raises(TypeError, match=r"fit\(\) got an unexpected " "keyword argument 'sample_weight'"): diff --git a/sklearn/covariance/_elliptic_envelope.py b/sklearn/covariance/_elliptic_envelope.py index 5ee4cdeeef96d..801611943f350 100644 --- a/sklearn/covariance/_elliptic_envelope.py +++ b/sklearn/covariance/_elliptic_envelope.py @@ -5,6 +5,7 @@ import numpy as np from . import MinCovDet from ..utils.validation import check_is_fitted, check_array +from ..utils.validation import _deprecate_positional_args from ..metrics import accuracy_score from ..base import OutlierMixin @@ -16,10 +17,10 @@ class EllipticEnvelope(OutlierMixin, MinCovDet): Parameters ---------- - store_precision : boolean, optional (default=True) + store_precision : bool, default=True Specify if the estimated precision is stored. - assume_centered : boolean, optional (default=False) + assume_centered : bool, default=False If True, the support of robust location and covariance estimates is computed, and a covariance estimate is recomputed from it, without centering the data. @@ -28,35 +29,34 @@ class EllipticEnvelope(OutlierMixin, MinCovDet): If False, the robust location and covariance are directly computed with the FastMCD algorithm without additional treatment. - support_fraction : float in (0., 1.), optional (default=None) + support_fraction : float, default=None The proportion of points to be included in the support of the raw MCD estimate. If None, the minimum value of support_fraction will be used within the algorithm: `[n_sample + n_features + 1] / 2`. + Range is (0, 1). - contamination : float in (0., 0.5), optional (default=0.1) + contamination : float, default=0.1 The amount of contamination of the data set, i.e. the proportion - of outliers in the data set. + of outliers in the data set. Range is (0, 0.5). - random_state : int, RandomState instance or None, optional (default=None) - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. + random_state : int or RandomState instance, default=None + Determines the pseudo random number generator for shuffling + the data. Pass an int for reproducible results across multiple function + calls. See :term: `Glossary `. Attributes ---------- - location_ : array-like, shape (n_features,) + location_ : ndarray of shape (n_features,) Estimated robust location - covariance_ : array-like, shape (n_features, n_features) + covariance_ : ndarray of shape (n_features, n_features) Estimated robust covariance matrix - precision_ : array-like, shape (n_features, n_features) + precision_ : ndarray of shape (n_features, n_features) Estimated pseudo inverse matrix. (stored only if store_precision is True) - support_ : array-like, shape (n_samples,) + support_ : ndarray of shape (n_samples,) A mask of the observations that have been used to compute the robust estimates of location and shape. @@ -67,6 +67,21 @@ class EllipticEnvelope(OutlierMixin, MinCovDet): such a way we obtain the expected number of outliers (samples with decision function < 0) in training. + raw_location_ : ndarray of shape (n_features,) + The raw robust estimated location before correction and re-weighting. + + raw_covariance_ : ndarray of shape (n_features, n_features) + The raw robust estimated covariance before correction and re-weighting. + + raw_support_ : ndarray of shape (n_samples,) + A mask of the observations that have been used to compute + the raw robust estimates of location and shape, before correction + and re-weighting. + + dist_ : ndarray of shape (n_samples,) + Mahalanobis distances of the training set (on which :meth:`fit` is + called) observations. + Examples -------- >>> import numpy as np @@ -102,9 +117,9 @@ class EllipticEnvelope(OutlierMixin, MinCovDet): .. [1] Rousseeuw, P.J., Van Driessen, K. "A fast algorithm for the minimum covariance determinant estimator" Technometrics 41(3), 212 (1999) - """ - def __init__(self, store_precision=True, assume_centered=False, + @_deprecate_positional_args + def __init__(self, *, store_precision=True, assume_centered=False, support_fraction=None, contamination=0.1, random_state=None): super().__init__( @@ -119,12 +134,11 @@ def fit(self, X, y=None): Parameters ---------- - X : numpy array or sparse matrix, shape (n_samples, n_features). - Training data + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training data. y : Ignored - not used, present for API consistency by convention. - + Not used, present for API consistency by convention. """ super().fit(X) self.offset_ = np.percentile(-self.dist_, 100. * self.contamination) @@ -135,17 +149,16 @@ def decision_function(self, X): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) + The data matrix. Returns ------- - - decision : array-like, shape (n_samples, ) + decision : ndarray of shape (n_samples, ) Decision function of the samples. It is equal to the shifted Mahalanobis distances. The threshold for being an outlier is 0, which ensures a compatibility with other outlier detection algorithms. - """ check_is_fitted(self) negative_mahal_dist = self.score_samples(X) @@ -156,11 +169,12 @@ def score_samples(self, X): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) + The data matrix. Returns ------- - negative_mahal_distances : array-like, shape (n_samples, ) + negative_mahal_distances : array-like of shape (n_samples,) Opposite of the Mahalanobis distances. """ check_is_fitted(self) @@ -173,11 +187,12 @@ def predict(self, X): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) + The data matrix. Returns ------- - is_inlier : array, shape (n_samples,) + is_inlier : ndarray of shape (n_samples,) Returns -1 for anomalies/outliers and +1 for inliers. """ X = check_array(X) @@ -196,19 +211,18 @@ def score(self, X, y, sample_weight=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Test samples. - y : array-like, shape (n_samples,) or (n_samples, n_outputs) + y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X. - sample_weight : array-like, shape (n_samples,), optional + sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns ------- score : float - Mean accuracy of self.predict(X) wrt. y. - + Mean accuracy of self.predict(X) w.r.t. y. """ return accuracy_score(y, self.predict(X), sample_weight=sample_weight) diff --git a/sklearn/covariance/_empirical_covariance.py b/sklearn/covariance/_empirical_covariance.py index 3a76abb326a26..c83dbc89697e1 100644 --- a/sklearn/covariance/_empirical_covariance.py +++ b/sklearn/covariance/_empirical_covariance.py @@ -18,6 +18,7 @@ from ..utils import check_array from ..utils.extmath import fast_logdet from ..metrics.pairwise import pairwise_distances +from ..utils.validation import _deprecate_positional_args def log_likelihood(emp_cov, precision): @@ -29,15 +30,16 @@ def log_likelihood(emp_cov, precision): Parameters ---------- - emp_cov : 2D ndarray (n_features, n_features) - Maximum Likelihood Estimator of covariance + emp_cov : ndarray of shape (n_features, n_features) + Maximum Likelihood Estimator of covariance. - precision : 2D ndarray (n_features, n_features) - The precision matrix of the covariance model to be tested + precision : ndarray of shape (n_features, n_features) + The precision matrix of the covariance model to be tested. Returns ------- - sample mean of the log-likelihood + log_likelihood_ : float + Sample mean of the log-likelihood. """ p = precision.shape[0] log_likelihood_ = - np.sum(emp_cov * precision) + fast_logdet(precision) @@ -52,10 +54,10 @@ def empirical_covariance(X, assume_centered=False): Parameters ---------- - X : ndarray, shape (n_samples, n_features) + X : ndarray of shape (n_samples, n_features) Data from which to compute the covariance estimate - assume_centered : boolean + assume_centered : bool, default=False If True, data will not be centered before computation. Useful when working with data whose mean is almost, but not exactly zero. @@ -63,9 +65,18 @@ def empirical_covariance(X, assume_centered=False): Returns ------- - covariance : 2D ndarray, shape (n_features, n_features) + covariance : ndarray of shape (n_features, n_features) Empirical covariance (Maximum Likelihood Estimator). + Examples + -------- + >>> from sklearn.covariance import empirical_covariance + >>> X = [[1,1,1],[1,1,1],[1,1,1], + ... [0,0,0],[0,0,0],[0,0,0]] + >>> empirical_covariance(X) + array([[0.25, 0.25, 0.25], + [0.25, 0.25, 0.25], + [0.25, 0.25, 0.25]]) """ X = np.asarray(X) if X.ndim == 1: @@ -92,10 +103,10 @@ class EmpiricalCovariance(BaseEstimator): Parameters ---------- - store_precision : bool + store_precision : bool, default=True Specifies if the estimated precision is stored. - assume_centered : bool + assume_centered : bool, default=False If True, data are not centered before computation. Useful when working with data whose mean is almost, but not exactly zero. @@ -103,13 +114,13 @@ class EmpiricalCovariance(BaseEstimator): Attributes ---------- - location_ : array-like, shape (n_features,) + location_ : ndarray of shape (n_features,) Estimated location, i.e. the estimated mean. - covariance_ : 2D ndarray, shape (n_features, n_features) + covariance_ : ndarray of shape (n_features, n_features) Estimated covariance matrix - precision_ : 2D ndarray, shape (n_features, n_features) + precision_ : ndarray of shape (n_features, n_features) Estimated pseudo-inverse matrix. (stored only if store_precision is True) @@ -132,7 +143,8 @@ class EmpiricalCovariance(BaseEstimator): array([0.0622..., 0.0193...]) """ - def __init__(self, store_precision=True, assume_centered=False): + @_deprecate_positional_args + def __init__(self, *, store_precision=True, assume_centered=False): self.store_precision = store_precision self.assume_centered = assume_centered @@ -144,10 +156,9 @@ def _set_covariance(self, covariance): Parameters ---------- - covariance : 2D ndarray, shape (n_features, n_features) + covariance : array-like of shape (n_features, n_features) Estimated covariance matrix to be stored, and from which precision is computed. - """ covariance = check_array(covariance) # set covariance @@ -163,9 +174,8 @@ def get_precision(self): Returns ------- - precision_ : array-like + precision_ : array-like of shape (n_features, n_features) The precision matrix associated to the current covariance object. - """ if self.store_precision: precision = self.precision_ @@ -183,15 +193,14 @@ def fit(self, X, y=None): Training data, where n_samples is the number of samples and n_features is the number of features. - y - not used, present for API consistence purpose. + y : Ignored + Not used, present for API consistence purpose. Returns ------- self : object - """ - X = check_array(X) + X = self._validate_data(X) if self.assume_centered: self.location_ = np.zeros(X.shape[1]) else: @@ -214,15 +223,14 @@ def score(self, X_test, y=None): X_test is assumed to be drawn from the same distribution than the data used in fit (including centering). - y - not used, present for API consistence purpose. + y : Ignored + Not used, present for API consistence purpose. Returns ------- res : float The likelihood of the data set with `self.covariance_` as an estimator of its covariance matrix. - """ # compute empirical covariance of the test set test_cov = empirical_covariance( @@ -242,26 +250,26 @@ def error_norm(self, comp_cov, norm='frobenius', scaling=True, comp_cov : array-like of shape (n_features, n_features) The covariance to compare with. - norm : str + norm : {"frobenius", "spectral"}, default="frobenius" The type of norm used to compute the error. Available error types: - 'frobenius' (default): sqrt(tr(A^t.A)) - 'spectral': sqrt(max(eigenvalues(A^t.A)) where A is the error ``(comp_cov - self.covariance_)``. - scaling : bool + scaling : bool, default=True If True (default), the squared error norm is divided by n_features. If False, the squared error norm is not rescaled. - squared : bool + squared : bool, default=True Whether to compute the squared error norm or the error norm. If True (default), the squared error norm is returned. If False, the error norm is returned. Returns ------- - The Mean Squared Error (in the sense of the Frobenius norm) between - `self` and `comp_cov` covariance estimators. - + result : float + The Mean Squared Error (in the sense of the Frobenius norm) between + `self` and `comp_cov` covariance estimators. """ # compute the error error = comp_cov - self.covariance_ @@ -296,9 +304,8 @@ def mahalanobis(self, X): Returns ------- - dist : array, shape = [n_samples,] + dist : ndarray of shape (n_samples,) Squared Mahalanobis distances of the observations. - """ precision = self.get_precision() # compute mahalanobis distances diff --git a/sklearn/covariance/_graph_lasso.py b/sklearn/covariance/_graph_lasso.py index c282d40c826bd..77ff9adb7fc0c 100644 --- a/sklearn/covariance/_graph_lasso.py +++ b/sklearn/covariance/_graph_lasso.py @@ -19,6 +19,7 @@ from ..exceptions import ConvergenceWarning from ..utils.validation import check_random_state, check_array +from ..utils.validation import _deprecate_positional_args from ..linear_model import _cd_fast as cd_fast from ..linear_model import lars_path_gram from ..model_selection import check_cv, cross_val_score @@ -58,16 +59,14 @@ def alpha_max(emp_cov): Parameters ---------- - emp_cov : 2D array, (n_features, n_features) - The sample covariance matrix + emp_cov : ndarray of shape (n_features, n_features) + The sample covariance matrix. Notes ----- - This results from the bound for the all the Lasso that are solved in GraphicalLasso: each time, the row of cov corresponds to Xy. As the bound for alpha is given by `max(abs(Xy))`, the result follows. - """ A = np.copy(emp_cov) A.flat[::A.shape[0] + 1] = 0 @@ -86,56 +85,57 @@ def graphical_lasso(emp_cov, alpha, cov_init=None, mode='cd', tol=1e-4, Parameters ---------- - emp_cov : 2D ndarray, shape (n_features, n_features) + emp_cov : ndarray of shape (n_features, n_features) Empirical covariance from which to compute the covariance estimate. - alpha : positive float + alpha : float The regularization parameter: the higher alpha, the more regularization, the sparser the inverse covariance. + Range is (0, inf]. - cov_init : 2D array (n_features, n_features), optional + cov_init : array of shape (n_features, n_features), default=None The initial guess for the covariance. - mode : {'cd', 'lars'} + mode : {'cd', 'lars'}, default='cd' The Lasso solver to use: coordinate descent or LARS. Use LARS for very sparse underlying graphs, where p > n. Elsewhere prefer cd which is more numerically stable. - tol : positive float, optional + tol : float, default=1e-4 The tolerance to declare convergence: if the dual gap goes below - this value, iterations are stopped. + this value, iterations are stopped. Range is (0, inf]. - enet_tol : positive float, optional + enet_tol : float, default=1e-4 The tolerance for the elastic net solver used to calculate the descent direction. This parameter controls the accuracy of the search direction for a given column update, not of the overall parameter estimate. Only - used for mode='cd'. + used for mode='cd'. Range is (0, inf]. - max_iter : integer, optional + max_iter : int, default=100 The maximum number of iterations. - verbose : boolean, optional + verbose : bool, default=False If verbose is True, the objective function and dual gap are printed at each iteration. - return_costs : boolean, optional + return_costs : bool, default=Flase If return_costs is True, the objective function and dual gap at each iteration are returned. - eps : float, optional + eps : float, default=eps The machine-precision regularization in the computation of the Cholesky diagonal factors. Increase this for very ill-conditioned - systems. + systems. Default is `np.finfo(np.float64).eps`. - return_n_iter : bool, optional + return_n_iter : bool, default=False Whether or not to return the number of iterations. Returns ------- - covariance : 2D ndarray, shape (n_features, n_features) + covariance : ndarray of shape (n_features, n_features) The estimated covariance matrix. - precision : 2D ndarray, shape (n_features, n_features) + precision : ndarray of shape (n_features, n_features) The estimated (sparse) precision matrix. costs : list of (objective, dual_gap) pairs @@ -157,7 +157,6 @@ def graphical_lasso(emp_cov, alpha, cov_init=None, mode='cd', tol=1e-4, One possible difference with the `glasso` R package is that the diagonal coefficients are not penalized. - """ _, n_features = emp_cov.shape if alpha == 0: @@ -285,33 +284,34 @@ class GraphicalLasso(EmpiricalCovariance): Parameters ---------- - alpha : positive float, default 0.01 + alpha : float, default=0.01 The regularization parameter: the higher alpha, the more regularization, the sparser the inverse covariance. + Range is (0, inf]. - mode : {'cd', 'lars'}, default 'cd' + mode : {'cd', 'lars'}, default='cd' The Lasso solver to use: coordinate descent or LARS. Use LARS for very sparse underlying graphs, where p > n. Elsewhere prefer cd which is more numerically stable. - tol : positive float, default 1e-4 + tol : float, default=1e-4 The tolerance to declare convergence: if the dual gap goes below - this value, iterations are stopped. + this value, iterations are stopped. Range is (0, inf]. - enet_tol : positive float, optional + enet_tol : float, default=1e-4 The tolerance for the elastic net solver used to calculate the descent direction. This parameter controls the accuracy of the search direction for a given column update, not of the overall parameter estimate. Only - used for mode='cd'. + used for mode='cd'. Range is (0, inf]. - max_iter : integer, default 100 + max_iter : int, default=100 The maximum number of iterations. - verbose : boolean, default False + verbose : bool, default=False If verbose is True, the objective function and dual gap are plotted at each iteration. - assume_centered : boolean, default False + assume_centered : bool, default=False If True, data are not centered before computation. Useful when working with data whose mean is almost, but not exactly zero. @@ -319,13 +319,13 @@ class GraphicalLasso(EmpiricalCovariance): Attributes ---------- - location_ : array-like, shape (n_features,) + location_ : ndarray of shape (n_features,) Estimated location, i.e. the estimated mean. - covariance_ : array-like, shape (n_features, n_features) + covariance_ : ndarray of shape (n_features, n_features) Estimated covariance matrix - precision_ : array-like, shape (n_features, n_features) + precision_ : ndarray of shape (n_features, n_features) Estimated pseudo inverse matrix. n_iter_ : int @@ -356,8 +356,8 @@ class GraphicalLasso(EmpiricalCovariance): -------- graphical_lasso, GraphicalLassoCV """ - - def __init__(self, alpha=.01, mode='cd', tol=1e-4, enet_tol=1e-4, + @_deprecate_positional_args + def __init__(self, alpha=.01, *, mode='cd', tol=1e-4, enet_tol=1e-4, max_iter=100, verbose=False, assume_centered=False): super().__init__(assume_centered=assume_centered) self.alpha = alpha @@ -372,13 +372,19 @@ def fit(self, X, y=None): Parameters ---------- - X : ndarray, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Data from which to compute the covariance estimate - y : (ignored) + + y : Ignored + Not used, present for API consistence purpose. + + Returns + ------- + self : object """ # Covariance does not make sense for a single feature - X = check_array(X, ensure_min_features=2, ensure_min_samples=2, - estimator=self) + X = self._validate_data(X, ensure_min_features=2, ensure_min_samples=2, + estimator=self) if self.assume_centered: self.location_ = np.zeros(X.shape[1]) @@ -402,49 +408,53 @@ def graphical_lasso_path(X, alphas, cov_init=None, X_test=None, mode='cd', Parameters ---------- - X : 2D ndarray, shape (n_samples, n_features) + X : ndarray of shape (n_samples, n_features) Data from which to compute the covariance estimate. - alphas : list of positive floats + alphas : array-like of shape (n_alphas,) The list of regularization parameters, decreasing order. - cov_init : 2D array (n_features, n_features), optional + cov_init : array of shape (n_features, n_features), default=None The initial guess for the covariance. - X_test : 2D array, shape (n_test_samples, n_features), optional + X_test : array of shape (n_test_samples, n_features), default=None Optional test matrix to measure generalisation error. - mode : {'cd', 'lars'} + mode : {'cd', 'lars'}, default='cd' The Lasso solver to use: coordinate descent or LARS. Use LARS for very sparse underlying graphs, where p > n. Elsewhere prefer cd which is more numerically stable. - tol : positive float, optional + tol : float, default=1e-4 The tolerance to declare convergence: if the dual gap goes below - this value, iterations are stopped. + this value, iterations are stopped. The tolerance must be a positive + number. - enet_tol : positive float, optional + enet_tol : float, default=1e-4 The tolerance for the elastic net solver used to calculate the descent direction. This parameter controls the accuracy of the search direction for a given column update, not of the overall parameter estimate. Only - used for mode='cd'. + used for mode='cd'. The tolerance must be a positive number. - max_iter : integer, optional - The maximum number of iterations. + max_iter : int, default=100 + The maximum number of iterations. This parameter should be a strictly + positive integer. - verbose : integer, optional + verbose : int or bool, default=False The higher the verbosity flag, the more information is printed during the fitting. Returns ------- - covariances_ : List of 2D ndarray, shape (n_features, n_features) + covariances_ : list of shape (n_alphas,) of ndarray of shape \ + (n_features, n_features) The estimated covariance matrices. - precisions_ : List of 2D ndarray, shape (n_features, n_features) + precisions_ : list of shape (n_alphas,) of ndarray of shape \ + (n_features, n_features) The estimated (sparse) precision matrices. - scores_ : List of float + scores_ : list of shape (n_alphas,), dtype=float The generalisation error (log-likelihood) on the test data. Returned only if test data is passed. """ @@ -500,17 +510,17 @@ class GraphicalLassoCV(GraphicalLasso): Parameters ---------- - alphas : integer, or list positive float, optional + alphas : int or array-like of shape (n_alphas,), dtype=float, default=4 If an integer is given, it fixes the number of points on the grids of alpha to be used. If a list is given, it gives the grid to be used. See the notes in the class docstring for - more details. + more details. Range is (0, inf] when floats given. - n_refinements : strictly positive integer + n_refinements : int, default=4 The number of times the grid is refined. Not used if explicit - values of alphas are passed. + values of alphas are passed. Range is [1, inf). - cv : int, cross-validation generator or an iterable, optional + cv : int, cross-validation generator or iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: @@ -527,36 +537,36 @@ class GraphicalLassoCV(GraphicalLasso): .. versionchanged:: 0.20 ``cv`` default value if None changed from 3-fold to 5-fold. - tol : positive float, optional + tol : float, default=1e-4 The tolerance to declare convergence: if the dual gap goes below - this value, iterations are stopped. + this value, iterations are stopped. Range is (0, inf]. - enet_tol : positive float, optional + enet_tol : float, default=1e-4 The tolerance for the elastic net solver used to calculate the descent direction. This parameter controls the accuracy of the search direction for a given column update, not of the overall parameter estimate. Only - used for mode='cd'. + used for mode='cd'. Range is (0, inf]. - max_iter : integer, optional + max_iter : int, default=100 Maximum number of iterations. - mode : {'cd', 'lars'} + mode : {'cd', 'lars'}, default='cd' The Lasso solver to use: coordinate descent or LARS. Use LARS for very sparse underlying graphs, where number of features is greater than number of samples. Elsewhere prefer cd which is more numerically stable. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None number of jobs to run in parallel. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - verbose : boolean, optional + verbose : bool, default=False If verbose is True, the objective function and duality gap are printed at each iteration. - assume_centered : boolean + assume_centered : bool, default=False If True, data are not centered before computation. Useful when working with data whose mean is almost, but not exactly zero. @@ -564,22 +574,22 @@ class GraphicalLassoCV(GraphicalLasso): Attributes ---------- - location_ : array-like, shape (n_features,) + location_ : ndarray of shape (n_features,) Estimated location, i.e. the estimated mean. - covariance_ : numpy.ndarray, shape (n_features, n_features) + covariance_ : ndarray of shape (n_features, n_features) Estimated covariance matrix. - precision_ : numpy.ndarray, shape (n_features, n_features) + precision_ : ndarray of shape (n_features, n_features) Estimated precision matrix (inverse covariance). alpha_ : float Penalization parameter selected. - cv_alphas_ : list of float + cv_alphas_ : list of shape (n_alphas,), dtype=float All penalization parameters explored. - grid_scores_ : 2D numpy.ndarray (n_alphas, n_folds) + grid_scores_ : ndarray of shape (n_alphas, n_folds) Log-likelihood score on left-out data across folds. n_iter_ : int @@ -622,8 +632,8 @@ class GraphicalLassoCV(GraphicalLasso): values of alpha then come out as missing values, but the optimum may be close to these missing values. """ - - def __init__(self, alphas=4, n_refinements=4, cv=None, tol=1e-4, + @_deprecate_positional_args + def __init__(self, *, alphas=4, n_refinements=4, cv=None, tol=1e-4, enet_tol=1e-4, max_iter=100, mode='cd', n_jobs=None, verbose=False, assume_centered=False): super().__init__( @@ -639,12 +649,18 @@ def fit(self, X, y=None): Parameters ---------- - X : ndarray, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Data from which to compute the covariance estimate - y : (ignored) + + y : Ignored + Not used, present for API consistence purpose. + + Returns + ------- + self : object """ # Covariance does not make sense for a single feature - X = check_array(X, ensure_min_features=2, estimator=self) + X = self._validate_data(X, ensure_min_features=2, estimator=self) if self.assume_centered: self.location_ = np.zeros(X.shape[1]) else: diff --git a/sklearn/covariance/_robust_covariance.py b/sklearn/covariance/_robust_covariance.py index 9c59f204a7636..73b36942682a1 100644 --- a/sklearn/covariance/_robust_covariance.py +++ b/sklearn/covariance/_robust_covariance.py @@ -17,6 +17,7 @@ from . import empirical_covariance, EmpiricalCovariance from ..utils.extmath import fast_logdet from ..utils import check_random_state, check_array +from ..utils.validation import _deprecate_positional_args # Minimum Covariance Determinant @@ -33,48 +34,49 @@ def c_step(X, n_support, remaining_iterations=30, initial_estimates=None, Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Data set in which we look for the n_support observations whose scatter matrix has minimum determinant. - n_support : int, > n_samples / 2 + n_support : int, Number of observations to compute the robust estimates of location - and covariance from. + and covariance from. This parameter must be greater than + `n_samples / 2`. - remaining_iterations : int, optional + remaining_iterations : int, default=30 Number of iterations to perform. According to [Rouseeuw1999]_, two iterations are sufficient to get close to the minimum, and we never need more than 30 to reach convergence. - initial_estimates : 2-tuple, optional + initial_estimates : tuple of shape (2,), default=None Initial estimates of location and shape from which to run the c_step procedure: - initial_estimates[0]: an initial location estimate - initial_estimates[1]: an initial covariance estimate - verbose : boolean, optional + verbose : bool, defaut=False Verbose mode. - cov_computation_method : callable, default empirical_covariance + cov_computation_method : callable, \ + default=:func:`sklearn.covariance.empirical_covariance` The function which will be used to compute the covariance. - Must return shape (n_features, n_features) + Must return array of shape (n_features, n_features). - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState instance, default=None + Determines the pseudo random number generator for shuffling the data. + Pass an int for reproducible results across multiple function calls. + See :term: `Glossary `. Returns ------- - location : array-like, shape (n_features,) + location : ndarray of shape (n_features,) Robust location estimates. - covariance : array-like, shape (n_features, n_features) + covariance : ndarray of shape (n_features, n_features) Robust covariance estimates. - support : array-like, shape (n_samples,) + support : ndarray of shape (n_samples,) A mask for the `n_support` observations whose scatter matrix has minimum determinant. @@ -83,7 +85,6 @@ def c_step(X, n_support, remaining_iterations=30, initial_estimates=None, .. [Rouseeuw1999] A Fast Algorithm for the Minimum Covariance Determinant Estimator, 1999, American Statistical Association and the American Society for Quality, TECHNOMETRICS - """ X = np.asarray(X) random_state = check_random_state(random_state) @@ -199,15 +200,17 @@ def select_candidates(X, n_support, n_trials, select=1, n_iter=30, Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Data (sub)set in which we look for the n_support purest observations. - n_support : int, [(n + p + 1)/2] < n_support < n + n_support : int The number of samples the pure data set must contain. + This parameter must be in the range `[(n + p + 1)/2] < n_support < n`. - n_trials : int, nb_trials > 0 or 2-tuple + n_trials : int or tuple of shape (2,) Number of different initial sets of observations from which to - run the algorithm. + run the algorithm. This parameter should be a strictly positive + integer. Instead of giving a number of trials to perform, one can provide a list of initial estimates that will be used to iteratively run c_step procedures. In this case: @@ -216,25 +219,27 @@ def select_candidates(X, n_support, n_trials, select=1, n_iter=30, - n_trials[1]: array-like, shape (n_trials, n_features, n_features) is the list of `n_trials` initial covariances estimates - select : int, int > 0 - Number of best candidates results to return. + select : int, default=1 + Number of best candidates results to return. This parameter must be + a strictly positive integer. - n_iter : int, nb_iter > 0 + n_iter : int, default=30 Maximum number of iterations for the c_step procedure. (2 is enough to be close to the final solution. "Never" exceeds 20). + This parameter must be a strictly positive integer. - verbose : boolean, default False + verbose : bool, default False Control the output verbosity. - cov_computation_method : callable, default empirical_covariance + cov_computation_method : callable, \ + default=:func:`sklearn.covariance.empirical_covariance` The function which will be used to compute the covariance. - Must return shape (n_features, n_features) + Must return an array of shape (n_features, n_features). - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState instance, default=None + Determines the pseudo random number generator for shuffling the data. + Pass an int for reproducible results across multiple function calls. + See :term: `Glossary `. See Also --------- @@ -242,15 +247,15 @@ def select_candidates(X, n_support, n_trials, select=1, n_iter=30, Returns ------- - best_locations : array-like, shape (select, n_features) + best_locations : ndarray of shape (select, n_features) The `select` location estimates computed from the `select` best supports found in the data set (`X`). - best_covariances : array-like, shape (select, n_features, n_features) + best_covariances : ndarray of shape (select, n_features, n_features) The `select` covariance estimates computed from the `select` best supports found in the data set (`X`). - best_supports : array-like, shape (select, n_samples) + best_supports : ndarray of shape (select, n_samples) The `select` best supports found in the data set (`X`). References @@ -258,7 +263,6 @@ def select_candidates(X, n_support, n_trials, select=1, n_iter=30, .. [RV] A Fast Algorithm for the Minimum Covariance Determinant Estimator, 1999, American Statistical Association and the American Society for Quality, TECHNOMETRICS - """ random_state = check_random_state(random_state) @@ -312,24 +316,37 @@ def fast_mcd(X, support_fraction=None, Parameters ---------- - X : array-like, shape (n_samples, n_features) - The data matrix, with p features and n samples. + X : array-like of shape (n_samples, n_features) + The data matrix, with p features and n samples. - support_fraction : float, 0 < support_fraction < 1 - The proportion of points to be included in the support of the raw - MCD estimate. Default is None, which implies that the minimum - value of support_fraction will be used within the algorithm: - `[n_sample + n_features + 1] / 2`. + support_fraction : float, default=None + The proportion of points to be included in the support of the raw + MCD estimate. Default is `None`, which implies that the minimum + value of `support_fraction` will be used within the algorithm: + `(n_sample + n_features + 1) / 2`. This parameter must be in the + range (0, 1). - cov_computation_method : callable, default empirical_covariance + cov_computation_method : callable, \ + default=:func:`sklearn.covariance.empirical_covariance` The function which will be used to compute the covariance. - Must return shape (n_features, n_features) + Must return an array of shape (n_features, n_features). + + random_state : int or RandomState instance, default=None + Determines the pseudo random number generator for shuffling the data. + Pass an int for reproducible results across multiple function calls. + See :term: `Glossary `. + + Returns + ------- + location : ndarray of shape (n_features,) + Robust location of the data. + + covariance : ndarray of shape (n_features, n_features) + Robust covariance of the features. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + support : ndarray of shape (n_samples,), dtype=bool + A mask of the observations that have been used to compute + the robust location and covariance estimates of the data set. Notes ----- @@ -356,19 +373,6 @@ def fast_mcd(X, support_fraction=None, .. [Butler1993] R. W. Butler, P. L. Davies and M. Jhun, Asymptotics For The Minimum Covariance Determinant Estimator, The Annals of Statistics, 1993, Vol. 21, No. 3, 1385-1400 - - Returns - ------- - location : array-like, shape (n_features,) - Robust location of the data. - - covariance : array-like, shape (n_features, n_features) - Robust covariance of the features. - - support : array-like, type boolean, shape (n_samples,) - A mask of the observations that have been used to compute - the robust location and covariance estimates of the data set. - """ random_state = check_random_state(random_state) @@ -524,10 +528,10 @@ class MinCovDet(EmpiricalCovariance): Parameters ---------- - store_precision : bool + store_precision : bool, default=True Specify if the estimated precision is stored. - assume_centered : bool + assume_centered : bool, default=False If True, the support of the robust location and the covariance estimates is computed, and a covariance estimate is recomputed from it, without centering the data. @@ -536,46 +540,46 @@ class MinCovDet(EmpiricalCovariance): If False, the robust location and covariance are directly computed with the FastMCD algorithm without additional treatment. - support_fraction : float, 0 < support_fraction < 1 + support_fraction : float, default=None The proportion of points to be included in the support of the raw MCD estimate. Default is None, which implies that the minimum value of support_fraction will be used within the algorithm: - [n_sample + n_features + 1] / 2 + `(n_sample + n_features + 1) / 2`. The parameter must be in the range + (0, 1). - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState instance, default=None + Determines the pseudo random number generator for shuffling the data. + Pass an int for reproducible results across multiple function calls. + See :term: `Glossary `. Attributes ---------- - raw_location_ : array-like, shape (n_features,) + raw_location_ : ndarray of shape (n_features,) The raw robust estimated location before correction and re-weighting. - raw_covariance_ : array-like, shape (n_features, n_features) + raw_covariance_ : ndarray of shape (n_features, n_features) The raw robust estimated covariance before correction and re-weighting. - raw_support_ : array-like, shape (n_samples,) + raw_support_ : ndarray of shape (n_samples,) A mask of the observations that have been used to compute the raw robust estimates of location and shape, before correction and re-weighting. - location_ : array-like, shape (n_features,) - Estimated robust location + location_ : ndarray of shape (n_features,) + Estimated robust location. - covariance_ : array-like, shape (n_features, n_features) - Estimated robust covariance matrix + covariance_ : ndarray of shape (n_features, n_features) + Estimated robust covariance matrix. - precision_ : array-like, shape (n_features, n_features) + precision_ : ndarray of shape (n_features, n_features) Estimated pseudo inverse matrix. (stored only if store_precision is True) - support_ : array-like, shape (n_samples,) + support_ : ndarray of shape (n_samples,) A mask of the observations that have been used to compute the robust estimates of location and shape. - dist_ : array-like, shape (n_samples,) + dist_ : ndarray of shape (n_samples,) Mahalanobis distances of the training set (on which :meth:`fit` is called) observations. @@ -608,11 +612,11 @@ class MinCovDet(EmpiricalCovariance): .. [ButlerDavies] R. W. Butler, P. L. Davies and M. Jhun, Asymptotics For The Minimum Covariance Determinant Estimator, The Annals of Statistics, 1993, Vol. 21, No. 3, 1385-1400 - """ _nonrobust_covariance = staticmethod(empirical_covariance) - def __init__(self, store_precision=True, assume_centered=False, + @_deprecate_positional_args + def __init__(self, *, store_precision=True, assume_centered=False, support_fraction=None, random_state=None): self.store_precision = store_precision self.assume_centered = assume_centered @@ -625,18 +629,17 @@ def fit(self, X, y=None): Parameters ---------- X : array-like of shape (n_samples, n_features) - Training data, where n_samples is the number of samples - and n_features is the number of features. + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. - y - not used, present for API consistence purpose. + y: Ignored + Not used, present for API consistence purpose. Returns ------- self : object - """ - X = check_array(X, ensure_min_samples=2, estimator='MinCovDet') + X = self._validate_data(X, ensure_min_samples=2, estimator='MinCovDet') random_state = check_random_state(self.random_state) n_samples, n_features = X.shape # check that the empirical covariance is full rank @@ -676,23 +679,22 @@ def correct_covariance(self, data): Parameters ---------- - data : array-like, shape (n_samples, n_features) + data : array-like of shape (n_samples, n_features) The data matrix, with p features and n samples. The data set must be the one which was used to compute the raw estimates. + Returns + ------- + covariance_corrected : ndarray of shape (n_features, n_features) + Corrected robust covariance estimate. + References ---------- .. [RVD] A Fast Algorithm for the Minimum Covariance Determinant Estimator, 1999, American Statistical Association and the American Society for Quality, TECHNOMETRICS - - Returns - ------- - covariance_corrected : array-like, shape (n_features, n_features) - Corrected robust covariance estimate. - """ # Check that the covariance of the support data is not equal to 0. @@ -717,30 +719,29 @@ def reweight_covariance(self, data): Parameters ---------- - data : array-like, shape (n_samples, n_features) + data : array-like of shape (n_samples, n_features) The data matrix, with p features and n samples. The data set must be the one which was used to compute the raw estimates. - References - ---------- - - .. [RVDriessen] A Fast Algorithm for the Minimum Covariance - Determinant Estimator, 1999, American Statistical Association - and the American Society for Quality, TECHNOMETRICS - Returns ------- - location_reweighted : array-like, shape (n_features, ) + location_reweighted : ndarray of shape (n_features,) Re-weighted robust location estimate. - covariance_reweighted : array-like, shape (n_features, n_features) + covariance_reweighted : ndarray of shape (n_features, n_features) Re-weighted robust covariance estimate. - support_reweighted : array-like, type boolean, shape (n_samples,) + support_reweighted : ndarray of shape (n_samples,), dtype=bool A mask of the observations that have been used to compute the re-weighted robust location and covariance estimates. + References + ---------- + + .. [RVDriessen] A Fast Algorithm for the Minimum Covariance + Determinant Estimator, 1999, American Statistical Association + and the American Society for Quality, TECHNOMETRICS """ n_samples, n_features = data.shape mask = self.dist_ < chi2(n_features).isf(0.025) diff --git a/sklearn/covariance/_shrunk_covariance.py b/sklearn/covariance/_shrunk_covariance.py index 9b01d3e7a9041..06e1b4f180347 100644 --- a/sklearn/covariance/_shrunk_covariance.py +++ b/sklearn/covariance/_shrunk_covariance.py @@ -18,6 +18,7 @@ from . import empirical_covariance, EmpiricalCovariance from ..utils import check_array +from ..utils.validation import _deprecate_positional_args # ShrunkCovariance estimator @@ -29,16 +30,16 @@ def shrunk_covariance(emp_cov, shrinkage=0.1): Parameters ---------- - emp_cov : array-like, shape (n_features, n_features) + emp_cov : array-like of shape (n_features, n_features) Covariance matrix to be shrunk - shrinkage : float, 0 <= shrinkage <= 1 + shrinkage : float, default=0.1 Coefficient in the convex combination used for the computation - of the shrunk estimate. + of the shrunk estimate. Range is [0, 1]. Returns ------- - shrunk_cov : array-like + shrunk_cov : ndarray of shape (n_features, n_features) Shrunk covariance. Notes @@ -48,7 +49,6 @@ def shrunk_covariance(emp_cov, shrinkage=0.1): (1 - shrinkage) * cov + shrinkage * mu * np.identity(n_features) where mu = trace(cov) / n_features - """ emp_cov = check_array(emp_cov) n_features = emp_cov.shape[0] @@ -67,28 +67,28 @@ class ShrunkCovariance(EmpiricalCovariance): Parameters ---------- - store_precision : boolean, default True + store_precision : bool, default=True Specify if the estimated precision is stored - assume_centered : boolean, default False + assume_centered : bool, default=False If True, data will not be centered before computation. Useful when working with data whose mean is almost, but not exactly zero. If False, data will be centered before computation. - shrinkage : float, 0 <= shrinkage <= 1, default 0.1 + shrinkage : float, default=0.1 Coefficient in the convex combination used for the computation - of the shrunk estimate. + of the shrunk estimate. Range is [0, 1]. Attributes ---------- - location_ : array-like, shape (n_features,) + location_ : ndarray of shape (n_features,) Estimated location, i.e. the estimated mean. - covariance_ : array-like, shape (n_features, n_features) + covariance_ : ndarray of shape (n_features, n_features) Estimated covariance matrix - precision_ : array-like, shape (n_features, n_features) + precision_ : ndarray of shape (n_features, n_features) Estimated pseudo inverse matrix. (stored only if store_precision is True) @@ -117,17 +117,17 @@ class ShrunkCovariance(EmpiricalCovariance): (1 - shrinkage) * cov + shrinkage * mu * np.identity(n_features) where mu = trace(cov) / n_features - """ - def __init__(self, store_precision=True, assume_centered=False, + @_deprecate_positional_args + def __init__(self, *, store_precision=True, assume_centered=False, shrinkage=0.1): super().__init__(store_precision=store_precision, assume_centered=assume_centered) self.shrinkage = shrinkage def fit(self, X, y=None): - """ Fits the shrunk covariance model - according to the given training data and parameters. + """Fit the shrunk covariance model according to the given training data + and parameters. Parameters ---------- @@ -135,15 +135,14 @@ def fit(self, X, y=None): Training data, where n_samples is the number of samples and n_features is the number of features. - y + y: Ignored not used, present for API consistence purpose. Returns ------- self : object - """ - X = check_array(X) + X = self._validate_data(X) # Not calling the parent object to fit, to avoid a potential # matrix inversion when setting the precision if self.assume_centered: @@ -167,16 +166,16 @@ def ledoit_wolf_shrinkage(X, assume_centered=False, block_size=1000): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Data from which to compute the Ledoit-Wolf shrunk covariance shrinkage. - assume_centered : bool + assume_centered : bool, default=False If True, data will not be centered before computation. Useful to work with data whose mean is significantly equal to zero but is not exactly zero. If False, data will be centered before computation. - block_size : int + block_size : int, default=1000 Size of the blocks into which the covariance matrix will be split. Returns @@ -192,7 +191,6 @@ def ledoit_wolf_shrinkage(X, assume_centered=False, block_size=1000): (1 - shrinkage) * cov + shrinkage * mu * np.identity(n_features) where mu = trace(cov) / n_features - """ X = np.asarray(X) # for only one feature, the result is the same whatever the shrinkage @@ -262,10 +260,10 @@ def ledoit_wolf(X, assume_centered=False, block_size=1000): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Data from which to compute the covariance estimate - assume_centered : boolean, default=False + assume_centered : bool, default=False If True, data will not be centered before computation. Useful to work with data whose mean is significantly equal to zero but is not exactly zero. @@ -277,7 +275,7 @@ def ledoit_wolf(X, assume_centered=False, block_size=1000): Returns ------- - shrunk_cov : array-like, shape (n_features, n_features) + shrunk_cov : ndarray of shape (n_features, n_features) Shrunk covariance. shrinkage : float @@ -291,7 +289,6 @@ def ledoit_wolf(X, assume_centered=False, block_size=1000): (1 - shrinkage) * cov + shrinkage * mu * np.identity(n_features) where mu = trace(cov) / n_features - """ X = np.asarray(X) # for only one feature, the result is the same whatever the shrinkage @@ -347,19 +344,19 @@ class LedoitWolf(EmpiricalCovariance): Attributes ---------- - location_ : array-like, shape (n_features,) + location_ : ndarray of shape (n_features,) Estimated location, i.e. the estimated mean. - covariance_ : array-like, shape (n_features, n_features) - Estimated covariance matrix + covariance_ : ndarray of shape (n_features, n_features) + Estimated covariance matrix. - precision_ : array-like, shape (n_features, n_features) + precision_ : ndarray of shape (n_features, n_features) Estimated pseudo inverse matrix. (stored only if store_precision is True) - shrinkage_ : float, 0 <= shrinkage <= 1 + shrinkage_ : float Coefficient in the convex combination used for the computation - of the shrunk estimate. + of the shrunk estimate. Range is [0, 1]. Examples -------- @@ -392,34 +389,33 @@ class LedoitWolf(EmpiricalCovariance): "A Well-Conditioned Estimator for Large-Dimensional Covariance Matrices", Ledoit and Wolf, Journal of Multivariate Analysis, Volume 88, Issue 2, February 2004, pages 365-411. - """ - def __init__(self, store_precision=True, assume_centered=False, + @_deprecate_positional_args + def __init__(self, *, store_precision=True, assume_centered=False, block_size=1000): super().__init__(store_precision=store_precision, assume_centered=assume_centered) self.block_size = block_size def fit(self, X, y=None): - """ Fits the Ledoit-Wolf shrunk covariance model - according to the given training data and parameters. + """Fit the Ledoit-Wolf shrunk covariance model according to the given + training data and parameters. Parameters ---------- X : array-like of shape (n_samples, n_features) - Training data, where n_samples is the number of samples - and n_features is the number of features. - y + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. + y : Ignored not used, present for API consistence purpose. Returns ------- self : object - """ # Not calling the parent object to fit, to avoid computing the # covariance matrix (and potentially the precision) - X = check_array(X) + X = self._validate_data(X) if self.assume_centered: self.location_ = np.zeros(X.shape[1]) else: @@ -440,10 +436,10 @@ def oas(X, assume_centered=False): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Data from which to compute the covariance estimate. - assume_centered : boolean + assume_centered : bool, default=False If True, data will not be centered before computation. Useful to work with data whose mean is significantly equal to zero but is not exactly zero. @@ -451,7 +447,7 @@ def oas(X, assume_centered=False): Returns ------- - shrunk_cov : array-like, shape (n_features, n_features) + shrunk_cov : array-like of shape (n_features, n_features) Shrunk covariance. shrinkage : float @@ -468,7 +464,6 @@ def oas(X, assume_centered=False): The formula we used to implement the OAS is slightly modified compared to the one given in the article. See :class:`OAS` for more details. - """ X = np.asarray(X) # for only one feature, the result is the same whatever the shrinkage @@ -528,16 +523,16 @@ class OAS(EmpiricalCovariance): Attributes ---------- - covariance_ : array-like, shape (n_features, n_features) + covariance_ : ndarray of shape (n_features, n_features) Estimated covariance matrix. - precision_ : array-like, shape (n_features, n_features) + precision_ : ndarray of shape (n_features, n_features) Estimated pseudo inverse matrix. (stored only if store_precision is True) - shrinkage_ : float, 0 <= shrinkage <= 1 + shrinkage_ : float coefficient in the convex combination used for the computation - of the shrunk estimate. + of the shrunk estimate. Range is [0, 1]. Notes ----- @@ -552,27 +547,25 @@ class OAS(EmpiricalCovariance): ---------- "Shrinkage Algorithms for MMSE Covariance Estimation" Chen et al., IEEE Trans. on Sign. Proc., Volume 58, Issue 10, October 2010. - """ def fit(self, X, y=None): - """ Fits the Oracle Approximating Shrinkage covariance model + """Fit the Oracle Approximating Shrinkage covariance model according to the given training data and parameters. Parameters ---------- X : array-like of shape (n_samples, n_features) - Training data, where n_samples is the number of samples - and n_features is the number of features. - y + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. + y : Ignored not used, present for API consistence purpose. Returns ------- self : object - """ - X = check_array(X) + X = self._validate_data(X) # Not calling the parent object to fit, to avoid computing the # covariance matrix (and potentially the precision) if self.assume_centered: diff --git a/sklearn/cross_decomposition/_cca.py b/sklearn/cross_decomposition/_cca.py index 80fa41bc44149..bd2e933339228 100644 --- a/sklearn/cross_decomposition/_cca.py +++ b/sklearn/cross_decomposition/_cca.py @@ -55,6 +55,9 @@ class CCA(_UnstableArchMixin, _PLS): y_rotations_ : array, [q, n_components] Y block to latents rotations. + coef_ : array of shape (p, q) + The coefficients of the linear model: ``Y = X coef_ + Err`` + n_iter_ : array-like Number of iterations of the NIPALS inner loop for each component. diff --git a/sklearn/cross_decomposition/_pls.py b/sklearn/cross_decomposition/_pls.py index a429872020ad0..88951d18468d8 100644 --- a/sklearn/cross_decomposition/_pls.py +++ b/sklearn/cross_decomposition/_pls.py @@ -40,6 +40,18 @@ def _nipals_twoblocks_inner_loop(X, Y, mode="A", max_iter=500, tol=1e-06, ite = 1 X_pinv = Y_pinv = None eps = np.finfo(X.dtype).eps + + if mode == "B": + # Uses condition from scipy<1.3 in pinv2 which was changed in + # https://github.com/scipy/scipy/pull/10067. In scipy 1.3, the + # condition was changed to depend on the largest singular value + X_t = X.dtype.char.lower() + Y_t = Y.dtype.char.lower() + factor = {'f': 1E3, 'd': 1E6} + + cond_X = factor[X_t] * eps + cond_Y = factor[Y_t] * eps + # Inner loop of the Wold algo. while True: # 1.1 Update u: the X weights @@ -47,7 +59,7 @@ def _nipals_twoblocks_inner_loop(X, Y, mode="A", max_iter=500, tol=1e-06, if X_pinv is None: # We use slower pinv2 (same as np.linalg.pinv) for stability # reasons - X_pinv = pinv2(X, check_finite=False) + X_pinv = pinv2(X, check_finite=False, cond=cond_X) x_weights = np.dot(X_pinv, y_score) else: # mode A # Mode A regress each X column on y_score @@ -64,7 +76,8 @@ def _nipals_twoblocks_inner_loop(X, Y, mode="A", max_iter=500, tol=1e-06, # 2.1 Update y_weights if mode == "B": if Y_pinv is None: - Y_pinv = pinv2(Y, check_finite=False) # compute once pinv(Y) + # compute once pinv(Y) + Y_pinv = pinv2(Y, check_finite=False, cond=cond_Y) y_weights = np.dot(Y_pinv, x_score) else: # Mode A regress each Y column on x_score @@ -264,8 +277,8 @@ def fit(self, X, Y): # copy since this will contains the residuals (deflated) matrices check_consistent_length(X, Y) - X = check_array(X, dtype=np.float64, copy=self.copy, - ensure_min_samples=2) + X = self._validate_data(X, dtype=np.float64, copy=self.copy, + ensure_min_samples=2) Y = check_array(Y, dtype=np.float64, copy=self.copy, ensure_2d=False) if Y.ndim == 1: Y = Y.reshape(-1, 1) @@ -517,6 +530,8 @@ class PLSRegression(_PLS): Read more in the :ref:`User Guide `. + .. versionadded:: 0.8 + Parameters ---------- n_components : int, (default 2) @@ -655,6 +670,8 @@ class PLSCanonical(_PLS): Read more in the :ref:`User Guide `. + .. versionadded:: 0.8 + Parameters ---------- n_components : int, (default 2). @@ -704,6 +721,9 @@ class PLSCanonical(_PLS): y_rotations_ : array, shape = [q, n_components] Y block to latents rotations. + coef_ : array of shape (p, q) + The coefficients of the linear model: ``Y = X coef_ + Err`` + n_iter_ : array-like Number of iterations of the NIPALS inner loop for each component. Not useful if the algorithm provided is "svd". @@ -797,6 +817,8 @@ class PLSSVD(TransformerMixin, BaseEstimator): Read more in the :ref:`User Guide `. + .. versionadded:: 0.8 + Parameters ---------- n_components : int, default 2 @@ -867,8 +889,8 @@ def fit(self, X, Y): """ # copy since this will contains the centered data check_consistent_length(X, Y) - X = check_array(X, dtype=np.float64, copy=self.copy, - ensure_min_samples=2) + X = self._validate_data(X, dtype=np.float64, copy=self.copy, + ensure_min_samples=2) Y = check_array(Y, dtype=np.float64, copy=self.copy, ensure_2d=False) if Y.ndim == 1: Y = Y.reshape(-1, 1) diff --git a/sklearn/cross_decomposition/tests/test_pls.py b/sklearn/cross_decomposition/tests/test_pls.py index 13c55fbd135d0..2d788a2cf6271 100644 --- a/sklearn/cross_decomposition/tests/test_pls.py +++ b/sklearn/cross_decomposition/tests/test_pls.py @@ -426,7 +426,6 @@ def test_pls_errors(): clf.fit, X, Y) -@pytest.mark.filterwarnings('ignore: The default value of multioutput') # 0.23 def test_pls_scaling(): # sanity check for scale=True n_samples = 1000 diff --git a/sklearn/datasets/__init__.py b/sklearn/datasets/__init__.py index 2377de2bfd189..e7c93bb180567 100644 --- a/sklearn/datasets/__init__.py +++ b/sklearn/datasets/__init__.py @@ -42,9 +42,9 @@ from ._samples_generator import make_gaussian_quantiles from ._samples_generator import make_biclusters from ._samples_generator import make_checkerboard -from ._svmlight_format import load_svmlight_file -from ._svmlight_format import load_svmlight_files -from ._svmlight_format import dump_svmlight_file +from ._svmlight_format_io import load_svmlight_file +from ._svmlight_format_io import load_svmlight_files +from ._svmlight_format_io import dump_svmlight_file from ._olivetti_faces import fetch_olivetti_faces from ._species_distributions import fetch_species_distributions from ._california_housing import fetch_california_housing diff --git a/sklearn/datasets/_base.py b/sklearn/datasets/_base.py index 9f33bc1f5fbf7..909470f980a5e 100644 --- a/sklearn/datasets/_base.py +++ b/sklearn/datasets/_base.py @@ -17,6 +17,7 @@ from ..utils import Bunch from ..utils import check_random_state +from ..utils import check_pandas_support import numpy as np @@ -67,6 +68,19 @@ def clear_data_home(data_home=None): shutil.rmtree(data_home) +def _convert_data_dataframe(caller_name, data, target, + feature_names, target_names): + pd = check_pandas_support('{} with as_frame=True'.format(caller_name)) + data_df = pd.DataFrame(data, columns=feature_names) + target_df = pd.DataFrame(target, columns=target_names) + combined_df = pd.concat([data_df, target_df], axis=1) + X = combined_df[feature_names] + y = combined_df[target_names] + if y.shape[1] == 1: + y = y.iloc[:, 0] + return combined_df, X, y + + def load_files(container_path, description=None, categories=None, load_content=True, shuffle=True, encoding=None, decode_error='strict', random_state=0): @@ -121,7 +135,7 @@ def load_files(container_path, description=None, categories=None, If None (default), load all the categories. If not None, list of category names to load (other categories ignored). - load_content : boolean, optional (default=True) + load_content : bool, optional (default=True) Whether to load or not the content of the different files. If true a 'data' attribute containing the text information is present in the data structure returned. If not, a filenames attribute gives the path to the @@ -142,19 +156,27 @@ def load_files(container_path, description=None, categories=None, contains characters not of the given `encoding`. Passed as keyword argument 'errors' to bytes.decode. - random_state : int, RandomState instance or None (default=0) + random_state : int, RandomState instance or None, default=0 Determines random number generation for dataset shuffling. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. Returns ------- - data : Bunch - Dictionary-like object, the interesting attributes are: either - data, the raw text data to learn, or 'filenames', the files - holding it, 'target', the classification labels (integer index), - 'target_names', the meaning of the labels, and 'DESCR', the full - description of the dataset. + data : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data : list of str + Only present when `load_content=True`. + The raw text data to learn. + target : ndarray + The target labels (integer index). + target_names : list + The names of target classes. + DESCR : str + The full description of the dataset. + filenames: ndarray + The filenames holding the dataset. """ target = [] target_names = [] @@ -246,7 +268,7 @@ def load_data(module_path, data_file_name): return data, target, target_names -def load_wine(return_X_y=False): +def load_wine(return_X_y=False, as_frame=False): """Load and return the wine dataset (classification). .. versionadded:: 0.18 @@ -266,17 +288,41 @@ def load_wine(return_X_y=False): Parameters ---------- - return_X_y : boolean, default=False. + return_X_y : bool, default=False. If True, returns ``(data, target)`` instead of a Bunch object. See below for more information about the `data` and `target` object. + as_frame : bool, default=False + If True, the data is a pandas DataFrame including columns with + appropriate dtypes (numeric). The target is + a pandas DataFrame or Series depending on the number of target columns. + If `return_X_y` is True, then (`data`, `target`) will be pandas + DataFrames or Series as described below. + + .. versionadded:: 0.23 + Returns ------- - data : Bunch - Dictionary-like object, the interesting attributes are: 'data', the - data to learn, 'target', the classification labels, 'target_names', the - meaning of the labels, 'feature_names', the meaning of the features, - and 'DESCR', the full description of the dataset. + data : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data : {ndarray, dataframe} of shape (178, 13) + The data matrix. If `as_frame=True`, `data` will be a pandas + DataFrame. + target: {ndarray, Series} of shape (178,) + The classification target. If `as_frame=True`, `target` will be + a pandas Series. + feature_names: list + The names of the dataset columns. + target_names: list + The names of target classes. + frame: DataFrame of shape (178, 14) + Only present when `as_frame=True`. DataFrame with `data` and + `target`. + + .. versionadded:: 0.23 + DESCR: str + The full description of the dataset. (data, target) : tuple if ``return_X_y`` is True @@ -302,28 +348,41 @@ def load_wine(return_X_y=False): with open(join(module_path, 'descr', 'wine_data.rst')) as rst_file: fdescr = rst_file.read() + feature_names = ['alcohol', + 'malic_acid', + 'ash', + 'alcalinity_of_ash', + 'magnesium', + 'total_phenols', + 'flavanoids', + 'nonflavanoid_phenols', + 'proanthocyanins', + 'color_intensity', + 'hue', + 'od280/od315_of_diluted_wines', + 'proline'] + + frame = None + target_columns = ['target', ] + if as_frame: + frame, data, target = _convert_data_dataframe("load_wine", + data, + target, + feature_names, + target_columns) + if return_X_y: return data, target - return Bunch(data=data, target=target, + return Bunch(data=data, + target=target, + frame=frame, target_names=target_names, DESCR=fdescr, - feature_names=['alcohol', - 'malic_acid', - 'ash', - 'alcalinity_of_ash', - 'magnesium', - 'total_phenols', - 'flavanoids', - 'nonflavanoid_phenols', - 'proanthocyanins', - 'color_intensity', - 'hue', - 'od280/od315_of_diluted_wines', - 'proline']) - - -def load_iris(return_X_y=False): + feature_names=feature_names) + + +def load_iris(return_X_y=False, as_frame=False): """Load and return the iris dataset (classification). The iris dataset is a classic and very easy multi-class classification @@ -341,21 +400,45 @@ def load_iris(return_X_y=False): Parameters ---------- - return_X_y : boolean, default=False. + return_X_y : bool, default=False. If True, returns ``(data, target)`` instead of a Bunch object. See below for more information about the `data` and `target` object. .. versionadded:: 0.18 + as_frame : bool, default=False + If True, the data is a pandas DataFrame including columns with + appropriate dtypes (numeric). The target is + a pandas DataFrame or Series depending on the number of target columns. + If `return_X_y` is True, then (`data`, `target`) will be pandas + DataFrames or Series as described below. + + .. versionadded:: 0.23 + Returns ------- - data : Bunch - Dictionary-like object, the interesting attributes are: - 'data', the data to learn, 'target', the classification labels, - 'target_names', the meaning of the labels, 'feature_names', the - meaning of the features, 'DESCR', the full description of - the dataset, 'filename', the physical location of - iris csv dataset (added in version `0.20`). + data : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data : {ndarray, dataframe} of shape (150, 4) + The data matrix. If `as_frame=True`, `data` will be a pandas + DataFrame. + target: {ndarray, Series} of shape (150,) + The classification target. If `as_frame=True`, `target` will be + a pandas Series. + feature_names: list + The names of the dataset columns. + target_names: list + The names of target classes. + frame: DataFrame of shape (150, 5) + Only present when `as_frame=True`. DataFrame with `data` and + `target`. + + .. versionadded:: 0.23 + DESCR: str + The full description of the dataset. + filename: str + The path to the location of the data. (data, target) : tuple if ``return_X_y`` is True @@ -387,18 +470,31 @@ def load_iris(return_X_y=False): with open(join(module_path, 'descr', 'iris.rst')) as rst_file: fdescr = rst_file.read() + feature_names = ['sepal length (cm)', 'sepal width (cm)', + 'petal length (cm)', 'petal width (cm)'] + + frame = None + target_columns = ['target', ] + if as_frame: + frame, data, target = _convert_data_dataframe("load_iris", + data, + target, + feature_names, + target_columns) + if return_X_y: return data, target - return Bunch(data=data, target=target, + return Bunch(data=data, + target=target, + frame=frame, target_names=target_names, DESCR=fdescr, - feature_names=['sepal length (cm)', 'sepal width (cm)', - 'petal length (cm)', 'petal width (cm)'], + feature_names=feature_names, filename=iris_csv_filename) -def load_breast_cancer(return_X_y=False): +def load_breast_cancer(return_X_y=False, as_frame=False): """Load and return the breast cancer wisconsin dataset (classification). The breast cancer dataset is a classic and very easy binary classification @@ -416,21 +512,45 @@ def load_breast_cancer(return_X_y=False): Parameters ---------- - return_X_y : boolean, default=False + return_X_y : bool, default=False If True, returns ``(data, target)`` instead of a Bunch object. See below for more information about the `data` and `target` object. .. versionadded:: 0.18 + as_frame : bool, default=False + If True, the data is a pandas DataFrame including columns with + appropriate dtypes (numeric). The target is + a pandas DataFrame or Series depending on the number of target columns. + If `return_X_y` is True, then (`data`, `target`) will be pandas + DataFrames or Series as described below. + + .. versionadded:: 0.23 + Returns ------- - data : Bunch - Dictionary-like object, the interesting attributes are: - 'data', the data to learn, 'target', the classification labels, - 'target_names', the meaning of the labels, 'feature_names', the - meaning of the features, and 'DESCR', the full description of - the dataset, 'filename', the physical location of - breast cancer csv dataset (added in version `0.20`). + data : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data : {ndarray, dataframe} of shape (569, 30) + The data matrix. If `as_frame=True`, `data` will be a pandas + DataFrame. + target: {ndarray, Series} of shape (569,) + The classification target. If `as_frame=True`, `target` will be + a pandas Series. + feature_names: list + The names of the dataset columns. + target_names: list + The names of target classes. + frame: DataFrame of shape (569, 31) + Only present when `as_frame=True`. DataFrame with `data` and + `target`. + + .. versionadded:: 0.23 + DESCR: str + The full description of the dataset. + filename: str + The path to the location of the data. (data, target) : tuple if ``return_X_y`` is True @@ -475,17 +595,28 @@ def load_breast_cancer(return_X_y=False): 'worst concavity', 'worst concave points', 'worst symmetry', 'worst fractal dimension']) + frame = None + target_columns = ['target', ] + if as_frame: + frame, data, target = _convert_data_dataframe("load_breast_cancer", + data, + target, + feature_names, + target_columns) + if return_X_y: return data, target - return Bunch(data=data, target=target, + return Bunch(data=data, + target=target, + frame=frame, target_names=target_names, DESCR=fdescr, feature_names=feature_names, filename=csv_filename) -def load_digits(n_class=10, return_X_y=False): +def load_digits(n_class=10, return_X_y=False, as_frame=False): """Load and return the digits dataset (classification). Each datapoint is a 8x8 image of a digit. @@ -505,20 +636,45 @@ def load_digits(n_class=10, return_X_y=False): n_class : integer, between 0 and 10, optional (default=10) The number of classes to return. - return_X_y : boolean, default=False. + return_X_y : bool, default=False. If True, returns ``(data, target)`` instead of a Bunch object. See below for more information about the `data` and `target` object. .. versionadded:: 0.18 + as_frame : bool, default=False + If True, the data is a pandas DataFrame including columns with + appropriate dtypes (numeric). The target is + a pandas DataFrame or Series depending on the number of target columns. + If `return_X_y` is True, then (`data`, `target`) will be pandas + DataFrames or Series as described below. + + .. versionadded:: 0.23 + Returns ------- - data : Bunch - Dictionary-like object, the interesting attributes are: - 'data', the data to learn, 'images', the images corresponding - to each sample, 'target', the classification labels for each - sample, 'target_names', the meaning of the labels, and 'DESCR', - the full description of the dataset. + data : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data : {ndarray, dataframe} of shape (1797, 64) + The flattened data matrix. If `as_frame=True`, `data` will be + a pandas DataFrame. + target: {ndarray, Series} of shape (1797,) + The classification target. If `as_frame=True`, `target` will be + a pandas Series. + feature_names: list + The names of the dataset columns. + target_names: list + The names of target classes. + frame: DataFrame of shape (1797, 65) + Only present when `as_frame=True`. DataFrame with `data` and + `target`. + + .. versionadded:: 0.23 + images: {ndarray} of shape (1797, 8, 8) + The raw image data. + DESCR: str + The full description of the dataset. (data, target) : tuple if ``return_X_y`` is True @@ -555,17 +711,32 @@ def load_digits(n_class=10, return_X_y=False): flat_data, target = flat_data[idx], target[idx] images = images[idx] + feature_names = ['pixel_{}_{}'.format(row_idx, col_idx) + for row_idx in range(8) + for col_idx in range(8)] + + frame = None + target_columns = ['target', ] + if as_frame: + frame, flat_data, target = _convert_data_dataframe("load_digits", + flat_data, + target, + feature_names, + target_columns) + if return_X_y: return flat_data, target return Bunch(data=flat_data, target=target, + frame=frame, + feature_names=feature_names, target_names=np.arange(10), images=images, DESCR=descr) -def load_diabetes(return_X_y=False): +def load_diabetes(return_X_y=False, as_frame=False): """Load and return the diabetes dataset (regression). ============== ================== @@ -579,20 +750,45 @@ def load_diabetes(return_X_y=False): Parameters ---------- - return_X_y : boolean, default=False. + return_X_y : bool, default=False. If True, returns ``(data, target)`` instead of a Bunch object. See below for more information about the `data` and `target` object. .. versionadded:: 0.18 + as_frame : bool, default=False + If True, the data is a pandas DataFrame including columns with + appropriate dtypes (numeric). The target is + a pandas DataFrame or Series depending on the number of target columns. + If `return_X_y` is True, then (`data`, `target`) will be pandas + DataFrames or Series as described below. + + .. versionadded:: 0.23 + Returns ------- - data : Bunch - Dictionary-like object, the interesting attributes are: - 'data', the data to learn, 'target', the regression target for each - sample, 'data_filename', the physical location - of diabetes data csv dataset, and 'target_filename', the physical - location of diabetes targets csv datataset (added in version `0.20`). + data : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data : {ndarray, dataframe} of shape (442, 10) + The data matrix. If `as_frame=True`, `data` will be a pandas + DataFrame. + target: {ndarray, Series} of shape (442,) + The regression target. If `as_frame=True`, `target` will be + a pandas Series. + feature_names: list + The names of the dataset columns. + frame: DataFrame of shape (442, 11) + Only present when `as_frame=True`. DataFrame with `data` and + `target`. + + .. versionadded:: 0.23 + DESCR: str + The full description of the dataset. + data_filename: str + The path to the location of the data. + target_filename: str + The path to the location of the target. (data, target) : tuple if ``return_X_y`` is True @@ -608,18 +804,34 @@ def load_diabetes(return_X_y=False): with open(join(module_path, 'descr', 'diabetes.rst')) as rst_file: fdescr = rst_file.read() + feature_names = ['age', 'sex', 'bmi', 'bp', + 's1', 's2', 's3', 's4', 's5', 's6'] + + frame = None + target_columns = ['target', ] + if as_frame: + frame, data, target = _convert_data_dataframe("load_diabetes", + data, + target, + feature_names, + target_columns) + if return_X_y: return data, target - return Bunch(data=data, target=target, DESCR=fdescr, - feature_names=['age', 'sex', 'bmi', 'bp', - 's1', 's2', 's3', 's4', 's5', 's6'], + return Bunch(data=data, + target=target, + frame=frame, + DESCR=fdescr, + feature_names=feature_names, data_filename=data_filename, target_filename=target_filename) -def load_linnerud(return_X_y=False): - """Load and return the linnerud dataset (multivariate regression). +def load_linnerud(return_X_y=False, as_frame=False): + """Load and return the physical excercise linnerud dataset. + + This dataset is suitable for multi-ouput regression tasks. ============== ============================ Samples total 20 @@ -632,23 +844,47 @@ def load_linnerud(return_X_y=False): Parameters ---------- - return_X_y : boolean, default=False. + return_X_y : bool, default=False. If True, returns ``(data, target)`` instead of a Bunch object. See below for more information about the `data` and `target` object. .. versionadded:: 0.18 + as_frame : bool, default=False + If True, the data is a pandas DataFrame including columns with + appropriate dtypes (numeric, string or categorical). The target is + a pandas DataFrame or Series depending on the number of target columns. + If `return_X_y` is True, then (`data`, `target`) will be pandas + DataFrames or Series as described below. + + .. versionadded:: 0.23 + Returns ------- - data : Bunch - Dictionary-like object, the interesting attributes are: 'data' and - 'target', the two multivariate datasets, with 'data' corresponding to - the exercise and 'target' corresponding to the physiological - measurements, as well as 'feature_names' and 'target_names'. - In addition, you will also have access to 'data_filename', - the physical location of linnerud data csv dataset, and - 'target_filename', the physical location of - linnerud targets csv datataset (added in version `0.20`). + data : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data : {ndarray, dataframe} of shape (20, 3) + The data matrix. If `as_frame=True`, `data` will be a pandas + DataFrame. + target: {ndarray, dataframe} of shape (20, 3) + The regression targets. If `as_frame=True`, `target` will be + a pandas DataFrame. + feature_names: list + The names of the dataset columns. + target_names: list + The names of the target columns. + frame: DataFrame of shape (20, 6) + Only present when `as_frame=True`. DataFrame with `data` and + `target`. + + .. versionadded:: 0.23 + DESCR: str + The full description of the dataset. + data_filename: str + The path to the location of the data. + target_filename: str + The path to the location of the target. (data, target) : tuple if ``return_X_y`` is True @@ -671,12 +907,23 @@ def load_linnerud(return_X_y=False): with open(dirname(__file__) + '/descr/linnerud.rst') as f: descr = f.read() + frame = None + if as_frame: + (frame, + data_exercise, + data_physiological) = _convert_data_dataframe("load_linnerud", + data_exercise, + data_physiological, + header_exercise, + header_physiological) if return_X_y: return data_exercise, data_physiological - return Bunch(data=data_exercise, feature_names=header_exercise, + return Bunch(data=data_exercise, + feature_names=header_exercise, target=data_physiological, target_names=header_physiological, + frame=frame, DESCR=descr, data_filename=data_filename, target_filename=target_filename) @@ -696,7 +943,7 @@ def load_boston(return_X_y=False): Parameters ---------- - return_X_y : boolean, default=False. + return_X_y : bool, default=False. If True, returns ``(data, target)`` instead of a Bunch object. See below for more information about the `data` and `target` object. @@ -704,12 +951,21 @@ def load_boston(return_X_y=False): Returns ------- - data : Bunch - Dictionary-like object, the interesting attributes are: - 'data', the data to learn, 'target', the regression targets, - 'DESCR', the full description of the dataset, - and 'filename', the physical location of boston - csv dataset (added in version `0.20`). + data : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data : ndarray of shape (506, 13) + The data matrix. + target : ndarray of shape (506, ) + The regression target. + filename : str + The physical location of boston csv dataset. + + .. versionadded:: 0.20 + DESCR : str + The full description of the dataset. + feature_names : ndarray + The names of features (data, target) : tuple if ``return_X_y`` is True @@ -768,10 +1024,15 @@ def load_sample_images(): Returns ------- - data : Bunch - Dictionary-like object with the following attributes : 'images', the - two sample images, 'filenames', the file names for the images, and - 'DESCR' the full description of the dataset. + data : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + images : list of ndarray of shape (427, 640, 3) + The two sample image. + filenames : list + The filenames for the images. + DESCR : str + The full description of the dataset. Examples -------- @@ -907,31 +1168,3 @@ def _fetch_remote(remote, dirname=None): "file may be corrupted.".format(file_path, checksum, remote.checksum)) return file_path - - -def _refresh_cache(files, compress): - # TODO: REMOVE in v0.23 - import joblib - msg = "sklearn.externals.joblib is deprecated in 0.21" - with warnings.catch_warnings(record=True) as warns: - data = tuple([joblib.load(f) for f in files]) - - refresh_needed = any([str(x.message).startswith(msg) for x in warns]) - - other_warns = [w for w in warns if not str(w.message).startswith(msg)] - for w in other_warns: - warnings.warn(message=w.message, category=w.category) - - if refresh_needed: - try: - for value, path in zip(data, files): - joblib.dump(value, path, compress=compress) - except IOError: - message = ("This dataset will stop being loadable in scikit-learn " - "version 0.23 because it references a deprecated " - "import path. Consider removing the following files " - "and allowing it to be cached anew:\n%s" - % ("\n".join(files))) - warnings.warn(message=message, category=FutureWarning) - - return data[0] if len(data) == 1 else data diff --git a/sklearn/datasets/_california_housing.py b/sklearn/datasets/_california_housing.py index bd02ff52ee19c..e3df2124aab2b 100644 --- a/sklearn/datasets/_california_housing.py +++ b/sklearn/datasets/_california_housing.py @@ -31,10 +31,10 @@ import joblib from . import get_data_home +from ._base import _convert_data_dataframe from ._base import _fetch_remote from ._base import _pkl_filepath from ._base import RemoteFileMetadata -from ._base import _refresh_cache from ..utils import Bunch # The original data can be found at: @@ -49,7 +49,7 @@ def fetch_california_housing(data_home=None, download_if_missing=True, - return_X_y=False): + return_X_y=False, as_frame=False): """Load the California housing dataset (regression). ============== ============== @@ -78,26 +78,40 @@ def fetch_california_housing(data_home=None, download_if_missing=True, .. versionadded:: 0.20 - Returns - ------- - dataset : dict-like object with the following attributes: - - dataset.data : ndarray, shape [20640, 8] - Each row corresponding to the 8 feature values in order. + as_frame : boolean, default=False + If True, the data is a pandas DataFrame including columns with + appropriate dtypes (numeric, string or categorical). The target is + a pandas DataFrame or Series depending on the number of target_columns. - dataset.target : numpy array of shape (20640,) - Each value corresponds to the average house value in units of 100,000. + .. versionadded:: 0.23 - dataset.feature_names : array of length 8 - Array of ordered feature names used in the dataset. - - dataset.DESCR : string - Description of the California housing dataset. + Returns + ------- + dataset : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data : ndarray, shape (20640, 8) + Each row corresponding to the 8 feature values in order. + If ``as_frame`` is True, ``data`` is a pandas object. + target : numpy array of shape (20640,) + Each value corresponds to the average + house value in units of 100,000. + If ``as_frame`` is True, ``target`` is a pandas object. + feature_names : list of length 8 + Array of ordered feature names used in the dataset. + DESCR : string + Description of the California housing dataset. (data, target) : tuple if ``return_X_y`` is True .. versionadded:: 0.20 + frame : pandas DataFrame + Only present when `as_frame=True`. DataFrame with ``data`` and + ``target``. + + .. versionadded:: 0.23 + Notes ----- @@ -130,9 +144,7 @@ def fetch_california_housing(data_home=None, download_if_missing=True, remove(archive_path) else: - cal_housing = _refresh_cache([filepath], 6) - # TODO: Revert to the following line in v0.23 - # cal_housing = joblib.load(filepath) + cal_housing = joblib.load(filepath) feature_names = ["MedInc", "HouseAge", "AveRooms", "AveBedrms", "Population", "AveOccup", "Latitude", "Longitude"] @@ -155,10 +167,24 @@ def fetch_california_housing(data_home=None, download_if_missing=True, with open(join(module_path, 'descr', 'california_housing.rst')) as dfile: descr = dfile.read() + X = data + y = target + + frame = None + target_names = ["MedHouseVal", ] + if as_frame: + frame, X, y = _convert_data_dataframe("fetch_california_housing", + data, + target, + feature_names, + target_names) + if return_X_y: - return data, target + return X, y - return Bunch(data=data, - target=target, + return Bunch(data=X, + target=y, + frame=frame, + target_names=target_names, feature_names=feature_names, DESCR=descr) diff --git a/sklearn/datasets/_covtype.py b/sklearn/datasets/_covtype.py index 233b19678f6de..6b23f913e05a7 100644 --- a/sklearn/datasets/_covtype.py +++ b/sklearn/datasets/_covtype.py @@ -25,7 +25,6 @@ from . import get_data_home from ._base import _fetch_remote from ._base import RemoteFileMetadata -from ._base import _refresh_cache from ..utils import Bunch from ._base import _pkl_filepath from ..utils import check_random_state @@ -66,7 +65,7 @@ def fetch_covtype(data_home=None, download_if_missing=True, If False, raise a IOError if the data is not locally available instead of trying to download the data from the source site. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset shuffling. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -82,17 +81,17 @@ def fetch_covtype(data_home=None, download_if_missing=True, Returns ------- - dataset : dict-like object with the following attributes: - - dataset.data : numpy array of shape (581012, 54) - Each row corresponds to the 54 features in the dataset. - - dataset.target : numpy array of shape (581012,) - Each value corresponds to one of the 7 forest covertypes with values - ranging between 1 to 7. - - dataset.DESCR : string - Description of the forest covertype dataset. + dataset : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data : numpy array of shape (581012, 54) + Each row corresponds to the 54 features in the dataset. + target : numpy array of shape (581012,) + Each value corresponds to one of + the 7 forest covertypes with values + ranging between 1 to 7. + DESCR : str + Description of the forest covertype dataset. (data, target) : tuple if ``return_X_y`` is True @@ -126,10 +125,8 @@ def fetch_covtype(data_home=None, download_if_missing=True, try: X, y except NameError: - X, y = _refresh_cache([samples_path, targets_path], 9) - # TODO: Revert to the following two lines in v0.23 - # X = joblib.load(samples_path) - # y = joblib.load(targets_path) + X = joblib.load(samples_path) + y = joblib.load(targets_path) if shuffle: ind = np.arange(X.shape[0]) diff --git a/sklearn/datasets/_kddcup99.py b/sklearn/datasets/_kddcup99.py index cfda9cfaeca84..c0ba00fa46f04 100644 --- a/sklearn/datasets/_kddcup99.py +++ b/sklearn/datasets/_kddcup99.py @@ -20,7 +20,6 @@ from ._base import _fetch_remote from . import get_data_home from ._base import RemoteFileMetadata -from ._base import _refresh_cache from ..utils import Bunch from ..utils import check_random_state from ..utils import shuffle as shuffle_method @@ -76,7 +75,7 @@ def fetch_kddcup99(subset=None, data_home=None, shuffle=False, shuffle : bool, default=False Whether to shuffle dataset. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset shuffling and for selection of abnormal samples if `subset='SA'`. Pass an int for reproducible output across multiple function calls. @@ -97,11 +96,15 @@ def fetch_kddcup99(subset=None, data_home=None, shuffle=False, Returns ------- - data : Bunch - Dictionary-like object, the interesting attributes are: - - 'data', the data to learn. - - 'target', the regression target for each sample. - - 'DESCR', a description of the dataset. + data : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data : ndarray of shape (494021, 41) + The data matrix to learn. + target : ndarray of shape (494021,) + The regression target for each sample. + DESCR : str + The full description of the dataset. (data, target) : tuple if ``return_X_y`` is True @@ -191,13 +194,15 @@ def _fetch_brute_kddcup99(data_home=None, Returns ------- - dataset : dict-like object with the following attributes: - dataset.data : numpy array of shape (494021, 41) + dataset : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data : numpy array of shape (494021, 41) Each row corresponds to the 41 features in the dataset. - dataset.target : numpy array of shape (494021,) + target : numpy array of shape (494021,) Each value corresponds to one of the 21 attack types or to the label 'normal.'. - dataset.DESCR : string + DESCR : string Description of the kddcup99 dataset. """ @@ -293,10 +298,8 @@ def _fetch_brute_kddcup99(data_home=None, try: X, y except NameError: - X, y = _refresh_cache([samples_path, targets_path], 0) - # TODO: Revert to the following two lines in v0.23 - # X = joblib.load(samples_path) - # y = joblib.load(targets_path) + X = joblib.load(samples_path) + y = joblib.load(targets_path) return Bunch(data=X, target=y) diff --git a/sklearn/datasets/_lfw.py b/sklearn/datasets/_lfw.py index 0cb65b3221039..b5efd68adbd1c 100644 --- a/sklearn/datasets/_lfw.py +++ b/sklearn/datasets/_lfw.py @@ -272,24 +272,23 @@ def fetch_lfw_people(data_home=None, funneled=True, resize=0.5, Returns ------- - dataset : dict-like object with the following attributes: - - dataset.data : numpy array of shape (13233, 2914) - Each row corresponds to a ravelled face image of original size 62 x 47 - pixels. Changing the ``slice_`` or resize parameters will change the - shape of the output. - - dataset.images : numpy array of shape (13233, 62, 47) - Each row is a face image corresponding to one of the 5749 people in - the dataset. Changing the ``slice_`` or resize parameters will change - the shape of the output. - - dataset.target : numpy array of shape (13233,) - Labels associated to each face image. Those labels range from 0-5748 - and correspond to the person IDs. - - dataset.DESCR : string - Description of the Labeled Faces in the Wild (LFW) dataset. + dataset : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data : numpy array of shape (13233, 2914) + Each row corresponds to a ravelled face image + of original size 62 x 47 pixels. + Changing the ``slice_`` or resize parameters will change the + shape of the output. + images : numpy array of shape (13233, 62, 47) + Each row is a face image corresponding to one of the 5749 people in + the dataset. Changing the ``slice_`` + or resize parameters will change the shape of the output. + target : numpy array of shape (13233,) + Labels associated to each face image. + Those labels range from 0-5748 and correspond to the person IDs. + DESCR : string + Description of the Labeled Faces in the Wild (LFW) dataset. (data, target) : tuple if ``return_X_y`` is True @@ -446,25 +445,25 @@ def fetch_lfw_pairs(subset='train', data_home=None, funneled=True, resize=0.5, Returns ------- - The data is returned as a Bunch object with the following attributes: - - data : numpy array of shape (2200, 5828). Shape depends on ``subset``. - Each row corresponds to 2 ravel'd face images of original size 62 x 47 - pixels. Changing the ``slice_``, ``resize`` or ``subset`` parameters - will change the shape of the output. - - pairs : numpy array of shape (2200, 2, 62, 47). Shape depends on ``subset`` - Each row has 2 face images corresponding to same or different person - from the dataset containing 5749 people. Changing the ``slice_``, - ``resize`` or ``subset`` parameters will change the shape of the - output. - - target : numpy array of shape (2200,). Shape depends on ``subset``. - Labels associated to each pair of images. The two label values being - different persons or the same person. - - DESCR : string - Description of the Labeled Faces in the Wild (LFW) dataset. + data : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data : ndarray of shape (2200, 5828). Shape depends on ``subset``. + Each row corresponds to 2 ravel'd face images + of original size 62 x 47 pixels. + Changing the ``slice_``, ``resize`` or ``subset`` parameters + will change the shape of the output. + pairs : ndarray of shape (2200, 2, 62, 47). Shape depends on ``subset`` + Each row has 2 face images corresponding + to same or different person from the dataset + containing 5749 people. Changing the ``slice_``, + ``resize`` or ``subset`` parameters will change the shape of the + output. + target : numpy array of shape (2200,). Shape depends on ``subset``. + Labels associated to each pair of images. + The two label values being different persons or the same person. + DESCR : string + Description of the Labeled Faces in the Wild (LFW) dataset. """ lfw_home, data_folder_path = _check_fetch_lfw( diff --git a/sklearn/datasets/_olivetti_faces.py b/sklearn/datasets/_olivetti_faces.py index 004f26b464836..d5f163d468214 100644 --- a/sklearn/datasets/_olivetti_faces.py +++ b/sklearn/datasets/_olivetti_faces.py @@ -24,7 +24,6 @@ from ._base import _fetch_remote from ._base import RemoteFileMetadata from ._base import _pkl_filepath -from ._base import _refresh_cache from ..utils import check_random_state, Bunch # The original data can be found at: @@ -61,7 +60,7 @@ def fetch_olivetti_faces(data_home=None, shuffle=False, random_state=0, If True the order of the dataset is shuffled to avoid having images of the same person grouped. - random_state : int, RandomState instance or None (default=0) + random_state : int, RandomState instance or None, default=0 Determines random number generation for dataset shuffling. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -78,15 +77,21 @@ def fetch_olivetti_faces(data_home=None, shuffle=False, random_state=0, Returns ------- - bunch : Bunch object with the following attributes: - - data: ndarray, shape (400, 4096). Each row corresponds to a ravelled - face image of original size 64 x 64 pixels. - - images : ndarray, shape (400, 64, 64). Each row is a face image - corresponding to one of the 40 subjects of the dataset. - - target : ndarray, shape (400,). Labels associated to each face image. - Those labels are ranging from 0-39 and correspond to the - Subject IDs. - - DESCR : string. Description of the modified Olivetti Faces Dataset. + data : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data: ndarray, shape (400, 4096) + Each row corresponds to a ravelled + face image of original size 64 x 64 pixels. + images : ndarray, shape (400, 64, 64) + Each row is a face image + corresponding to one of the 40 subjects of the dataset. + target : ndarray, shape (400,) + Labels associated to each face image. + Those labels are ranging from 0-39 and correspond to the + Subject IDs. + DESCR : str + Description of the modified Olivetti Faces Dataset. (data, target) : tuple if `return_X_y=True` .. versionadded:: 0.22 @@ -110,9 +115,7 @@ def fetch_olivetti_faces(data_home=None, shuffle=False, random_state=0, joblib.dump(faces, filepath, compress=6) del mfile else: - faces = _refresh_cache([filepath], 6) - # TODO: Revert to the following line in v0.23 - # faces = joblib.load(filepath) + faces = joblib.load(filepath) # We want floating point data, but float32 is enough (there is only # one byte of precision in the original uint8s anyway) diff --git a/sklearn/datasets/_openml.py b/sklearn/datasets/_openml.py index 2c2b194f9ef71..cef0e6cb1f411 100644 --- a/sklearn/datasets/_openml.py +++ b/sklearn/datasets/_openml.py @@ -579,8 +579,8 @@ def fetch_openml(name=None, version='active', data_id=None, data_home=None, Returns ------- - data : Bunch - Dictionary-like object, with attributes: + data : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. data : np.array, scipy.sparse.csr_matrix of floats, or pandas DataFrame The feature matrix. Categorical features are encoded as ordinals. diff --git a/sklearn/datasets/_rcv1.py b/sklearn/datasets/_rcv1.py index 53edc9a3407d8..4f1c5cc4af199 100644 --- a/sklearn/datasets/_rcv1.py +++ b/sklearn/datasets/_rcv1.py @@ -22,8 +22,7 @@ from ._base import _pkl_filepath from ._base import _fetch_remote from ._base import RemoteFileMetadata -from ._base import _refresh_cache -from ._svmlight_format import load_svmlight_files +from ._svmlight_format_io import load_svmlight_files from ..utils import shuffle as shuffle_ from ..utils import Bunch @@ -111,7 +110,7 @@ def fetch_rcv1(data_home=None, subset='all', download_if_missing=True, If False, raise a IOError if the data is not locally available instead of trying to download the data from the source site. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset shuffling. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -128,23 +127,20 @@ def fetch_rcv1(data_home=None, subset='all', download_if_missing=True, Returns ------- - dataset : dict-like object with the following attributes: - - dataset.data : scipy csr array, dtype np.float64, shape (804414, 47236) - The array has 0.16% of non zero values. - - dataset.target : scipy csr array, dtype np.uint8, shape (804414, 103) - Each sample has a value of 1 in its categories, and 0 in others. - The array has 3.15% of non zero values. - - dataset.sample_id : numpy array, dtype np.uint32, shape (804414,) - Identification number of each sample, as ordered in dataset.data. - - dataset.target_names : numpy array, dtype object, length (103) - Names of each target (RCV1 topics), as ordered in dataset.target. - - dataset.DESCR : string - Description of the RCV1 dataset. + dataset : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data : scipy csr array, dtype np.float64, shape (804414, 47236) + The array has 0.16% of non zero values. + target : scipy csr array, dtype np.uint8, shape (804414, 103) + Each sample has a value of 1 in its categories, and 0 in others. + The array has 3.15% of non zero values. + sample_id : numpy array, dtype np.uint32, shape (804414,) + Identification number of each sample, as ordered in dataset.data. + target_names : numpy array, dtype object, length (103) + Names of each target (RCV1 topics), as ordered in dataset.target. + DESCR : string + Description of the RCV1 dataset. (data, target) : tuple if ``return_X_y`` is True @@ -190,10 +186,8 @@ def fetch_rcv1(data_home=None, subset='all', download_if_missing=True, f.close() remove(f.name) else: - X, sample_id = _refresh_cache([samples_path, sample_id_path], 9) - # TODO: Revert to the following two lines in v0.23 - # X = joblib.load(samples_path) - # sample_id = joblib.load(sample_id_path) + X = joblib.load(samples_path) + sample_id = joblib.load(sample_id_path) # load target (y), categories, and sample_id_bis if download_if_missing and (not exists(sample_topics_path) or @@ -246,10 +240,8 @@ def fetch_rcv1(data_home=None, subset='all', download_if_missing=True, joblib.dump(y, sample_topics_path, compress=9) joblib.dump(categories, topics_path, compress=9) else: - y, categories = _refresh_cache([sample_topics_path, topics_path], 9) - # TODO: Revert to the following two lines in v0.23 - # y = joblib.load(sample_topics_path) - # categories = joblib.load(topics_path) + y = joblib.load(sample_topics_path) + categories = joblib.load(topics_path) if subset == 'all': pass diff --git a/sklearn/datasets/_samples_generator.py b/sklearn/datasets/_samples_generator.py index 5d18b46711489..62ef492f42f5e 100644 --- a/sklearn/datasets/_samples_generator.py +++ b/sklearn/datasets/_samples_generator.py @@ -124,7 +124,7 @@ def make_classification(n_samples=100, n_features=20, n_informative=2, shuffle : boolean, optional (default=True) Shuffle the samples and the features. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -320,7 +320,7 @@ def make_multilabel_classification(n_samples=100, n_features=20, n_classes=5, probabilities of features given classes, from which the data was drawn. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -342,6 +342,17 @@ def make_multilabel_classification(n_samples=100, n_features=20, n_classes=5, Only returned if ``return_distributions=True``. """ + if n_classes < 1: + raise ValueError( + "'n_classes' should be an integer greater than 0. Got {} instead." + .format(n_classes) + ) + if length < 1: + raise ValueError( + "'length' should be an integer greater than 0. Got {} instead." + .format(length) + ) + generator = check_random_state(random_state) p_c = generator.rand(n_classes) p_c /= p_c.sum() @@ -426,7 +437,7 @@ def make_hastie_10_2(n_samples=12000, random_state=None): n_samples : int, optional (default=12000) The number of samples. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -517,7 +528,7 @@ def make_regression(n_samples=100, n_features=100, n_informative=10, coef : boolean, optional (default=False) If True, the coefficients of the underlying linear model are returned. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -591,9 +602,12 @@ def make_circles(n_samples=100, shuffle=True, noise=None, random_state=None, Parameters ---------- - n_samples : int, optional (default=100) - The total number of points generated. If odd, the inner circle will - have one point more than the outer circle. + n_samples : int or two-element tuple, optional (default=100) + If int, it is the total number of points generated. + For odd numbers, the inner circle will have one point more than the + outer circle. + If two-element tuple, number of points in outer circle and inner + circle. shuffle : bool, optional (default=True) Whether to shuffle the samples. @@ -601,7 +615,7 @@ def make_circles(n_samples=100, shuffle=True, noise=None, random_state=None, noise : double or None (default=None) Standard deviation of Gaussian noise added to the data. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset shuffling and noise. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -621,8 +635,15 @@ def make_circles(n_samples=100, shuffle=True, noise=None, random_state=None, if factor >= 1 or factor < 0: raise ValueError("'factor' has to be between 0 and 1.") - n_samples_out = n_samples // 2 - n_samples_in = n_samples - n_samples_out + if isinstance(n_samples, numbers.Integral): + n_samples_out = n_samples // 2 + n_samples_in = n_samples - n_samples_out + else: + try: + n_samples_out, n_samples_in = n_samples + except ValueError: + raise ValueError('`n_samples` can be either an int or ' + 'a two-element tuple.') generator = check_random_state(random_state) # so as not to have the first point = last point, we set endpoint=False @@ -654,8 +675,9 @@ def make_moons(n_samples=100, shuffle=True, noise=None, random_state=None): Parameters ---------- - n_samples : int, optional (default=100) - The total number of points generated. + n_samples : int or two-element tuple, optional (default=100) + If int, the total number of points generated. + If two-element tuple, number of points in each of two moons. shuffle : bool, optional (default=True) Whether to shuffle the samples. @@ -663,7 +685,7 @@ def make_moons(n_samples=100, shuffle=True, noise=None, random_state=None): noise : double or None (default=None) Standard deviation of Gaussian noise added to the data. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset shuffling and noise. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -677,8 +699,15 @@ def make_moons(n_samples=100, shuffle=True, noise=None, random_state=None): The integer labels (0 or 1) for class membership of each sample. """ - n_samples_out = n_samples // 2 - n_samples_in = n_samples - n_samples_out + if isinstance(n_samples, numbers.Integral): + n_samples_out = n_samples // 2 + n_samples_in = n_samples - n_samples_out + else: + try: + n_samples_out, n_samples_in = n_samples + except ValueError: + raise ValueError('`n_samples` can be either an int or ' + 'a two-element tuple.') generator = check_random_state(random_state) @@ -702,7 +731,8 @@ def make_moons(n_samples=100, shuffle=True, noise=None, random_state=None): def make_blobs(n_samples=100, n_features=2, centers=None, cluster_std=1.0, - center_box=(-10.0, 10.0), shuffle=True, random_state=None): + center_box=(-10.0, 10.0), shuffle=True, random_state=None, + return_centers=False): """Generate isotropic Gaussian blobs for clustering. Read more in the :ref:`User Guide `. @@ -735,11 +765,16 @@ def make_blobs(n_samples=100, n_features=2, centers=None, cluster_std=1.0, shuffle : boolean, optional (default=True) Shuffle the samples. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. + return_centers : bool, optional (default=False) + If True, then return the centers of each cluster + + .. versionadded:: 0.23 + Returns ------- X : array of shape [n_samples, n_features] @@ -748,6 +783,10 @@ def make_blobs(n_samples=100, n_features=2, centers=None, cluster_std=1.0, y : array of shape [n_samples] The integer labels for cluster membership of each sample. + centers : array, shape [n_centers, n_features] + The centers of each cluster. Only returned if + ``return_centers=True``. + Examples -------- >>> from sklearn.datasets import make_blobs @@ -840,7 +879,10 @@ def make_blobs(n_samples=100, n_features=2, centers=None, cluster_std=1.0, X = X[indices] y = y[indices] - return X, y + if return_centers: + return X, y, centers + else: + return X, y def make_friedman1(n_samples=100, n_features=10, noise=0.0, random_state=None): @@ -872,7 +914,7 @@ def make_friedman1(n_samples=100, n_features=10, noise=0.0, random_state=None): noise : float, optional (default=0.0) The standard deviation of the gaussian noise applied to the output. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset noise. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -933,7 +975,7 @@ def make_friedman2(n_samples=100, noise=0.0, random_state=None): noise : float, optional (default=0.0) The standard deviation of the gaussian noise applied to the output. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset noise. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -998,7 +1040,7 @@ def make_friedman3(n_samples=100, noise=0.0, random_state=None): noise : float, optional (default=0.0) The standard deviation of the gaussian noise applied to the output. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset noise. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -1074,7 +1116,7 @@ def make_low_rank_matrix(n_samples=100, n_features=100, effective_rank=10, The relative importance of the fat noisy tail of the singular values profile. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -1127,7 +1169,7 @@ def make_sparse_coded_signal(n_samples, n_components, n_features, n_nonzero_coefs : int number of active (non-zero) coefficients in each sample - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -1186,7 +1228,7 @@ def make_sparse_uncorrelated(n_samples=100, n_features=10, random_state=None): n_features : int, optional (default=10) The number of features. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -1226,7 +1268,7 @@ def make_spd_matrix(n_dim, random_state=None): n_dim : int The matrix dimension. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -1275,7 +1317,7 @@ def make_sparse_spd_matrix(dim=1, alpha=0.95, norm_diag=False, largest_coef : float between 0 and 1, optional (default=0.9) The value of the largest coefficient. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -1336,7 +1378,7 @@ def make_swiss_roll(n_samples=100, noise=0.0, random_state=None): noise : float, optional (default=0.0) The standard deviation of the gaussian noise. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -1388,7 +1430,7 @@ def make_s_curve(n_samples=100, noise=0.0, random_state=None): noise : float, optional (default=0.0) The standard deviation of the gaussian noise. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -1451,7 +1493,7 @@ def make_gaussian_quantiles(mean=None, cov=1., n_samples=100, shuffle : boolean, optional (default=True) Shuffle the samples. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -1539,7 +1581,7 @@ def make_biclusters(shape, n_clusters, noise=0.0, minval=10, shuffle : boolean, optional (default=True) Shuffle the samples. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -1630,7 +1672,7 @@ def make_checkerboard(shape, n_clusters, noise=0.0, minval=10, shuffle : boolean, optional (default=True) Shuffle the samples. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. diff --git a/sklearn/datasets/_species_distributions.py b/sklearn/datasets/_species_distributions.py index 99dc192af755b..7f621d1de74eb 100644 --- a/sklearn/datasets/_species_distributions.py +++ b/sklearn/datasets/_species_distributions.py @@ -51,7 +51,6 @@ from ._base import RemoteFileMetadata from ..utils import Bunch from ._base import _pkl_filepath -from ._base import _refresh_cache # The original data can be found at: # https://biodiversityinformatics.amnh.org/open_source/maxent/samples.zip @@ -156,31 +155,28 @@ def fetch_species_distributions(data_home=None, Returns ------- - The data is returned as a Bunch object with the following attributes: - - coverages : array, shape = [14, 1592, 1212] - These represent the 14 features measured at each point of the map grid. - The latitude/longitude values for the grid are discussed below. - Missing data is represented by the value -9999. - - train : record array, shape = (1624,) - The training points for the data. Each point has three fields: - - - train['species'] is the species name - - train['dd long'] is the longitude, in degrees - - train['dd lat'] is the latitude, in degrees - - test : record array, shape = (620,) - The test points for the data. Same format as the training data. - - Nx, Ny : integers - The number of longitudes (x) and latitudes (y) in the grid - - x_left_lower_corner, y_left_lower_corner : floats - The (x,y) position of the lower-left corner, in degrees - - grid_size : float - The spacing between points of the grid, in degrees + data : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + coverages : array, shape = [14, 1592, 1212] + These represent the 14 features measured + at each point of the map grid. + The latitude/longitude values for the grid are discussed below. + Missing data is represented by the value -9999. + train : record array, shape = (1624,) + The training points for the data. Each point has three fields: + + - train['species'] is the species name + - train['dd long'] is the longitude, in degrees + - train['dd lat'] is the latitude, in degrees + test : record array, shape = (620,) + The test points for the data. Same format as the training data. + Nx, Ny : integers + The number of longitudes (x) and latitudes (y) in the grid + x_left_lower_corner, y_left_lower_corner : floats + The (x,y) position of the lower-left corner, in degrees + grid_size : float + The spacing between points of the grid, in degrees References ---------- @@ -260,8 +256,6 @@ def fetch_species_distributions(data_home=None, **extra_params) joblib.dump(bunch, archive_path, compress=9) else: - bunch = _refresh_cache([archive_path], 9) - # TODO: Revert to the following line in v0.23 - # bunch = joblib.load(archive_path) + bunch = joblib.load(archive_path) return bunch diff --git a/sklearn/datasets/_svmlight_format.py b/sklearn/datasets/_svmlight_format_io.py similarity index 99% rename from sklearn/datasets/_svmlight_format.py rename to sklearn/datasets/_svmlight_format_io.py index d344b310be995..91bb35ff2ec75 100644 --- a/sklearn/datasets/_svmlight_format.py +++ b/sklearn/datasets/_svmlight_format_io.py @@ -453,8 +453,10 @@ def dump_svmlight_file(X, y, f, zero_based=True, comment=None, query_id=None, Xval = check_array(X, accept_sparse='csr') if Xval.shape[0] != yval.shape[0]: - raise ValueError("X.shape[0] and y.shape[0] should be the same, got" - " %r and %r instead." % (Xval.shape[0], yval.shape[0])) + raise ValueError( + "X.shape[0] and y.shape[0] should be the same, got" + " %r and %r instead." % (Xval.shape[0], yval.shape[0]) + ) # We had some issues with CSR matrices with unsorted indices (e.g. #1501), # so sort them here, but first make sure we don't modify the user's X. diff --git a/sklearn/datasets/_twenty_newsgroups.py b/sklearn/datasets/_twenty_newsgroups.py index 5d43aa7c558ad..ebbd191069c49 100644 --- a/sklearn/datasets/_twenty_newsgroups.py +++ b/sklearn/datasets/_twenty_newsgroups.py @@ -184,7 +184,7 @@ def fetch_20newsgroups(data_home=None, subset='train', categories=None, make the assumption that the samples are independent and identically distributed (i.i.d.), such as stochastic gradient descent. - random_state : int, RandomState instance or None (default) + random_state : int, RandomState instance, default=None Determines random number generation for dataset shuffling. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -214,13 +214,19 @@ def fetch_20newsgroups(data_home=None, subset='train', categories=None, Returns ------- - bunch : Bunch object with the following attribute: - - data: list, length [n_samples] - - target: array, shape [n_samples] - - filenames: list, length [n_samples] - - DESCR: a description of the dataset. - - target_names: a list of categories of the returned data, - length [n_classes]. This depends on the `categories` parameter. + bunch : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data : list, length [n_samples] + The data list to learn. + target: array, shape [n_samples] + The target labels. + filenames: list, length [n_samples] + The path to the location of the data. + DESCR: str + The full description of the dataset. + target_names: list, length [n_classes] + The names of target classes. (data, target) : tuple if `return_X_y=True` .. versionadded:: 0.22 @@ -384,12 +390,17 @@ def fetch_20newsgroups_vectorized(subset="train", remove=(), data_home=None, Returns ------- - bunch : Bunch object with the following attribute: - - bunch.data: sparse matrix, shape [n_samples, n_features] - - bunch.target: array, shape [n_samples] - - bunch.target_names: a list of categories of the returned data, - length [n_classes]. - - bunch.DESCR: a description of the dataset. + bunch : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + data: sparse matrix, shape [n_samples, n_features] + The data matrix to learn. + target: array, shape [n_samples] + The target labels. + target_names: list, length [n_classes] + The names of target classes. + DESCR: str + The full description of the dataset. (data, target) : tuple if ``return_X_y`` is True diff --git a/sklearn/datasets/descr/diabetes.rst b/sklearn/datasets/descr/diabetes.rst index f2adc8d192b6c..771b3e5fe282a 100644 --- a/sklearn/datasets/descr/diabetes.rst +++ b/sklearn/datasets/descr/diabetes.rst @@ -17,16 +17,16 @@ quantitative measure of disease progression one year after baseline. :Target: Column 11 is a quantitative measure of disease progression one year after baseline :Attribute Information: - - Age - - Sex - - Body mass index - - Average blood pressure - - S1 - - S2 - - S3 - - S4 - - S5 - - S6 + - age age in years + - sex + - bmi body mass index + - bp average blood pressure + - s1 tc, T-Cells (a type of white blood cells) + - s2 ldl, low-density lipoproteins + - s3 hdl, high-density lipoproteins + - s4 tch, thyroid stimulating hormone + - s5 ltg, lamotrigine + - s6 glu, blood sugar level Note: Each of these 10 feature variables have been mean centered and scaled by the standard deviation times `n_samples` (i.e. the sum of squares of each column totals 1). diff --git a/sklearn/datasets/descr/linnerud.rst b/sklearn/datasets/descr/linnerud.rst index 5585b50a7e42b..55eda902448d9 100644 --- a/sklearn/datasets/descr/linnerud.rst +++ b/sklearn/datasets/descr/linnerud.rst @@ -9,14 +9,16 @@ Linnerrud dataset :Number of Attributes: 3 :Missing Attribute Values: None -The Linnerud dataset constains two small dataset: +The Linnerud dataset is a multi-output regression dataset. It consists of three +excercise (data) and three physiological (target) variables collected from +twenty middle-aged men in a fitness club: -- *physiological* - CSV containing 20 observations on 3 exercise variables: +- *physiological* - CSV containing 20 observations on 3 physiological variables: Weight, Waist and Pulse. - -- *exercise* - CSV containing 20 observations on 3 physiological variables: +- *exercise* - CSV containing 20 observations on 3 exercise variables: Chins, Situps and Jumps. .. topic:: References - * Tenenhaus, M. (1998). La regression PLS: theorie et pratique. Paris: Editions Technic. + * Tenenhaus, M. (1998). La regression PLS: theorie et pratique. Paris: + Editions Technic. diff --git a/sklearn/datasets/tests/conftest.py b/sklearn/datasets/tests/conftest.py new file mode 100644 index 0000000000000..fdb9516e62a27 --- /dev/null +++ b/sklearn/datasets/tests/conftest.py @@ -0,0 +1,75 @@ +""" Network tests are only run, if data is already locally available, +or if download is specifically requested by environment variable.""" +import builtins +from os import environ +import pytest +from sklearn.datasets import fetch_20newsgroups +from sklearn.datasets import fetch_20newsgroups_vectorized +from sklearn.datasets import fetch_california_housing +from sklearn.datasets import fetch_covtype +from sklearn.datasets import fetch_kddcup99 +from sklearn.datasets import fetch_olivetti_faces +from sklearn.datasets import fetch_rcv1 + + +def _wrapped_fetch(f, dataset_name): + """ Fetch dataset (download if missing and requested by environment) """ + download_if_missing = environ.get('SKLEARN_SKIP_NETWORK_TESTS', '1') == '0' + + def wrapped(*args, **kwargs): + kwargs['download_if_missing'] = download_if_missing + try: + return f(*args, **kwargs) + except IOError: + pytest.skip("Download {} to run this test".format(dataset_name)) + return wrapped + + +@pytest.fixture +def fetch_20newsgroups_fxt(): + return _wrapped_fetch(fetch_20newsgroups, dataset_name='20newsgroups') + + +@pytest.fixture +def fetch_20newsgroups_vectorized_fxt(): + return _wrapped_fetch(fetch_20newsgroups_vectorized, + dataset_name='20newsgroups_vectorized') + + +@pytest.fixture +def fetch_california_housing_fxt(): + return _wrapped_fetch(fetch_california_housing, + dataset_name='california_housing') + + +@pytest.fixture +def fetch_covtype_fxt(): + return _wrapped_fetch(fetch_covtype, dataset_name='covtype') + + +@pytest.fixture +def fetch_kddcup99_fxt(): + return _wrapped_fetch(fetch_kddcup99, dataset_name='kddcup99') + + +@pytest.fixture +def fetch_olivetti_faces_fxt(): + return _wrapped_fetch(fetch_olivetti_faces, dataset_name='olivetti_faces') + + +@pytest.fixture +def fetch_rcv1_fxt(): + return _wrapped_fetch(fetch_rcv1, dataset_name='rcv1') + + +@pytest.fixture +def hide_available_pandas(monkeypatch): + """ Pretend pandas was not installed. """ + import_orig = builtins.__import__ + + def mocked_import(name, *args, **kwargs): + if name == 'pandas': + raise ImportError() + return import_orig(name, *args, **kwargs) + + monkeypatch.setattr(builtins, '__import__', mocked_import) diff --git a/sklearn/datasets/tests/test_20news.py b/sklearn/datasets/tests/test_20news.py index 15cb49c44b0e5..f800a49238ec1 100644 --- a/sklearn/datasets/tests/test_20news.py +++ b/sklearn/datasets/tests/test_20news.py @@ -1,25 +1,21 @@ -"""Test the 20news downloader, if the data is available.""" +"""Test the 20news downloader, if the data is available, +or if specifically requested via environment variable +(e.g. for travis cron job).""" from functools import partial import numpy as np import scipy.sparse as sp -from sklearn.utils._testing import SkipTest, assert_allclose_dense_sparse +from sklearn.utils._testing import assert_allclose_dense_sparse from sklearn.datasets.tests.test_common import check_return_X_y - -from sklearn import datasets from sklearn.preprocessing import normalize -def test_20news(): - try: - data = datasets.fetch_20newsgroups( - subset='all', download_if_missing=False, shuffle=False) - except IOError: - raise SkipTest("Download 20 newsgroups to run this test") +def test_20news(fetch_20newsgroups_fxt): + data = fetch_20newsgroups_fxt(subset='all', shuffle=False) # Extract a reduced dataset - data2cats = datasets.fetch_20newsgroups( + data2cats = fetch_20newsgroups_fxt( subset='all', categories=data.target_names[-1:-3:-1], shuffle=False) # Check that the ordering of the target_names is the same # as the ordering in the full dataset @@ -40,72 +36,53 @@ def test_20news(): assert entry1 == entry2 # check that return_X_y option - X, y = datasets.fetch_20newsgroups( - subset='all', shuffle=False, return_X_y=True - ) + X, y = fetch_20newsgroups_fxt(subset='all', shuffle=False, return_X_y=True) assert len(X) == len(data.data) assert y.shape == data.target.shape -def test_20news_length_consistency(): +def test_20news_length_consistency(fetch_20newsgroups_fxt): """Checks the length consistencies within the bunch This is a non-regression test for a bug present in 0.16.1. """ - try: - data = datasets.fetch_20newsgroups( - subset='all', download_if_missing=False, shuffle=False) - except IOError: - raise SkipTest("Download 20 newsgroups to run this test") # Extract the full dataset - data = datasets.fetch_20newsgroups(subset='all') + data = fetch_20newsgroups_fxt(subset='all') assert len(data['data']) == len(data.data) assert len(data['target']) == len(data.target) assert len(data['filenames']) == len(data.filenames) -def test_20news_vectorized(): - try: - datasets.fetch_20newsgroups(subset='all', - download_if_missing=False) - except IOError: - raise SkipTest("Download 20 newsgroups to run this test") - +def test_20news_vectorized(fetch_20newsgroups_vectorized_fxt): # test subset = train - bunch = datasets.fetch_20newsgroups_vectorized(subset="train") + bunch = fetch_20newsgroups_vectorized_fxt(subset="train") assert sp.isspmatrix_csr(bunch.data) assert bunch.data.shape == (11314, 130107) assert bunch.target.shape[0] == 11314 assert bunch.data.dtype == np.float64 # test subset = test - bunch = datasets.fetch_20newsgroups_vectorized(subset="test") + bunch = fetch_20newsgroups_vectorized_fxt(subset="test") assert sp.isspmatrix_csr(bunch.data) assert bunch.data.shape == (7532, 130107) assert bunch.target.shape[0] == 7532 assert bunch.data.dtype == np.float64 # test return_X_y option - fetch_func = partial(datasets.fetch_20newsgroups_vectorized, subset='test') + fetch_func = partial(fetch_20newsgroups_vectorized_fxt, subset='test') check_return_X_y(bunch, fetch_func) # test subset = all - bunch = datasets.fetch_20newsgroups_vectorized(subset='all') + bunch = fetch_20newsgroups_vectorized_fxt(subset='all') assert sp.isspmatrix_csr(bunch.data) assert bunch.data.shape == (11314 + 7532, 130107) assert bunch.target.shape[0] == 11314 + 7532 assert bunch.data.dtype == np.float64 -def test_20news_normalization(): - try: - X = datasets.fetch_20newsgroups_vectorized(normalize=False, - download_if_missing=False) - X_ = datasets.fetch_20newsgroups_vectorized(normalize=True, - download_if_missing=False) - except IOError: - raise SkipTest("Download 20 newsgroups to run this test") - +def test_20news_normalization(fetch_20newsgroups_vectorized_fxt): + X = fetch_20newsgroups_vectorized_fxt(normalize=False) + X_ = fetch_20newsgroups_vectorized_fxt(normalize=True) X_norm = X_['data'][:100] X = X['data'][:100] diff --git a/sklearn/datasets/tests/test_base.py b/sklearn/datasets/tests/test_base.py index 3a0ad41ced969..a58bdc9ed644d 100644 --- a/sklearn/datasets/tests/test_base.py +++ b/sklearn/datasets/tests/test_base.py @@ -24,8 +24,9 @@ from sklearn.datasets import load_boston from sklearn.datasets import load_wine from sklearn.utils import Bunch -from sklearn.datasets._base import _refresh_cache from sklearn.datasets.tests.test_common import check_return_X_y +from sklearn.datasets.tests.test_common import check_as_frame +from sklearn.datasets.tests.test_common import check_pandas_dependency_message from sklearn.externals._pilutil import pillow_installed @@ -233,6 +234,33 @@ def test_load_breast_cancer(): check_return_X_y(res, partial(load_breast_cancer)) +@pytest.mark.parametrize("loader_func, data_dtype, target_dtype", [ + (load_breast_cancer, np.float64, np.int64), + (load_diabetes, np.float64, np.float64), + (load_digits, np.float64, np.int64), + (load_iris, np.float64, np.int64), + (load_linnerud, np.float64, np.float64), + (load_wine, np.float64, np.int64), +]) +def test_toy_dataset_as_frame(loader_func, data_dtype, target_dtype): + default_result = loader_func() + check_as_frame(default_result, partial(loader_func), + expected_data_dtype=data_dtype, + expected_target_dtype=target_dtype) + + +@pytest.mark.parametrize("loader_func", [ + load_breast_cancer, + load_diabetes, + load_digits, + load_iris, + load_linnerud, + load_wine, +]) +def test_toy_dataset_as_frame_no_pandas(loader_func): + check_pandas_dependency_message(loader_func) + + def test_load_boston(): res = load_boston() assert res.data.shape == (506, 13) @@ -277,55 +305,3 @@ def test_bunch_dir(): # check that dir (important for autocomplete) shows attributes data = load_iris() assert "data" in dir(data) - - -def test_refresh_cache(monkeypatch): - # uses pytests monkeypatch fixture - # https://docs.pytest.org/en/latest/monkeypatch.html - - def _load_warn(*args, **kwargs): - # raise the warning from "externals.joblib.__init__.py" - # this is raised when a file persisted by the old joblib is loaded now - msg = ("sklearn.externals.joblib is deprecated in 0.21 and will be " - "removed in 0.23. Please import this functionality directly " - "from joblib, which can be installed with: pip install joblib. " - "If this warning is raised when loading pickled models, you " - "may need to re-serialize those models with scikit-learn " - "0.21+.") - warnings.warn(msg, FutureWarning) - return 0 - - def _load_warn_unrelated(*args, **kwargs): - warnings.warn("unrelated warning", FutureWarning) - return 0 - - def _dump_safe(*args, **kwargs): - pass - - def _dump_raise(*args, **kwargs): - # this happens if the file is read-only and joblib.dump fails to write - # on it. - raise IOError() - - # test if the dataset spesific warning is raised if load raises the joblib - # warning, and dump fails to dump with new joblib - monkeypatch.setattr(joblib, "load", _load_warn) - monkeypatch.setattr(joblib, "dump", _dump_raise) - msg = "This dataset will stop being loadable in scikit-learn" - with pytest.warns(FutureWarning, match=msg): - _refresh_cache('test', 0) - - # make sure no warning is raised if load raises the warning, but dump - # manages to dump the new data - monkeypatch.setattr(joblib, "load", _load_warn) - monkeypatch.setattr(joblib, "dump", _dump_safe) - with pytest.warns(None) as warns: - _refresh_cache('test', 0) - assert len(warns) == 0 - - # test if an unrelated warning is still passed through and not suppressed - # by _refresh_cache - monkeypatch.setattr(joblib, "load", _load_warn_unrelated) - monkeypatch.setattr(joblib, "dump", _dump_safe) - with pytest.warns(FutureWarning, match="unrelated warning"): - _refresh_cache('test', 0) diff --git a/sklearn/datasets/tests/test_california_housing.py b/sklearn/datasets/tests/test_california_housing.py index ef45226c01f02..a8c5514e2ec73 100644 --- a/sklearn/datasets/tests/test_california_housing.py +++ b/sklearn/datasets/tests/test_california_housing.py @@ -1,26 +1,37 @@ -"""Test the california_housing loader. +"""Test the california_housing loader, if the data is available, +or if specifically requested via environment variable +(e.g. for travis cron job).""" +import pytest -Skipped if california_housing is not already downloaded to data_home. -""" - -from sklearn.datasets import fetch_california_housing -from sklearn.utils._testing import SkipTest from sklearn.datasets.tests.test_common import check_return_X_y from functools import partial -def fetch(*args, **kwargs): - return fetch_california_housing(*args, download_if_missing=False, **kwargs) - - -def test_fetch(): - try: - data = fetch() - except IOError: - raise SkipTest("California housing dataset can not be loaded.") +def test_fetch(fetch_california_housing_fxt): + data = fetch_california_housing_fxt() assert((20640, 8) == data.data.shape) assert((20640, ) == data.target.shape) # test return_X_y option - fetch_func = partial(fetch) + fetch_func = partial(fetch_california_housing_fxt) check_return_X_y(data, fetch_func) + + +def test_fetch_asframe(fetch_california_housing_fxt): + pd = pytest.importorskip('pandas') + bunch = fetch_california_housing_fxt(as_frame=True) + frame = bunch.frame + assert hasattr(bunch, 'frame') is True + assert frame.shape == (20640, 9) + assert isinstance(bunch.data, pd.DataFrame) + assert isinstance(bunch.target, pd.Series) + + +def test_pandas_dependency_message(fetch_california_housing_fxt, + hide_available_pandas): + # Check that pandas is imported lazily and that an informative error + # message is raised when pandas is missing: + expected_msg = ('fetch_california_housing with as_frame=True' + ' requires pandas') + with pytest.raises(ImportError, match=expected_msg): + fetch_california_housing_fxt(as_frame=True) diff --git a/sklearn/datasets/tests/test_common.py b/sklearn/datasets/tests/test_common.py index 6abce207ca920..15963e3c90141 100644 --- a/sklearn/datasets/tests/test_common.py +++ b/sklearn/datasets/tests/test_common.py @@ -1,5 +1,20 @@ """Test loaders for common functionality. """ +import pytest +import numpy as np + + +def check_pandas_dependency_message(fetch_func): + try: + import pandas # noqa + pytest.skip("This test requires pandas to be not installed") + except ImportError: + # Check that pandas is imported lazily and that an informative error + # message is raised when pandas is missing: + expected_msg = ('{} with as_frame=True requires pandas' + .format(fetch_func.__name__)) + with pytest.raises(ImportError, match=expected_msg): + fetch_func(as_frame=True) def check_return_X_y(bunch, fetch_func_partial): @@ -7,3 +22,22 @@ def check_return_X_y(bunch, fetch_func_partial): assert isinstance(X_y_tuple, tuple) assert X_y_tuple[0].shape == bunch.data.shape assert X_y_tuple[1].shape == bunch.target.shape + + +def check_as_frame(bunch, fetch_func_partial, + expected_data_dtype=None, expected_target_dtype=None): + pd = pytest.importorskip('pandas') + frame_bunch = fetch_func_partial(as_frame=True) + assert hasattr(frame_bunch, 'frame') + assert isinstance(frame_bunch.frame, pd.DataFrame) + assert isinstance(frame_bunch.data, pd.DataFrame) + assert frame_bunch.data.shape == bunch.data.shape + if frame_bunch.target.ndim > 1: + assert isinstance(frame_bunch.target, pd.DataFrame) + else: + assert isinstance(frame_bunch.target, pd.Series) + assert frame_bunch.target.shape[0] == bunch.target.shape[0] + if expected_data_dtype is not None: + assert np.all(frame_bunch.data.dtypes == expected_data_dtype) + if expected_target_dtype is not None: + assert np.all(frame_bunch.target.dtypes == expected_target_dtype) diff --git a/sklearn/datasets/tests/test_covtype.py b/sklearn/datasets/tests/test_covtype.py index 1127b8114c5e7..d966e6c3890d0 100644 --- a/sklearn/datasets/tests/test_covtype.py +++ b/sklearn/datasets/tests/test_covtype.py @@ -1,25 +1,14 @@ -"""Test the covtype loader. +"""Test the covtype loader, if the data is available, +or if specifically requested via environment variable +(e.g. for travis cron job).""" -Skipped if covtype is not already downloaded to data_home. -""" - -from sklearn.datasets import fetch_covtype -from sklearn.utils._testing import SkipTest from sklearn.datasets.tests.test_common import check_return_X_y from functools import partial -def fetch(*args, **kwargs): - return fetch_covtype(*args, download_if_missing=False, **kwargs) - - -def test_fetch(): - try: - data1 = fetch(shuffle=True, random_state=42) - except IOError: - raise SkipTest("Covertype dataset can not be loaded.") - - data2 = fetch(shuffle=True, random_state=37) +def test_fetch(fetch_covtype_fxt): + data1 = fetch_covtype_fxt(shuffle=True, random_state=42) + data2 = fetch_covtype_fxt(shuffle=True, random_state=37) X1, X2 = data1['data'], data2['data'] assert (581012, 54) == X1.shape @@ -32,5 +21,5 @@ def test_fetch(): assert (X1.shape[0],) == y2.shape # test return_X_y option - fetch_func = partial(fetch) + fetch_func = partial(fetch_covtype_fxt) check_return_X_y(data1, fetch_func) diff --git a/sklearn/datasets/tests/test_kddcup99.py b/sklearn/datasets/tests/test_kddcup99.py index 6d371e5a8e6f0..899abd2bcb153 100644 --- a/sklearn/datasets/tests/test_kddcup99.py +++ b/sklearn/datasets/tests/test_kddcup99.py @@ -1,55 +1,46 @@ -"""Test kddcup99 loader. Only 'percent10' mode is tested, as the full data -is too big to use in unit-testing. +"""Test kddcup99 loader, if the data is available, +or if specifically requested via environment variable +(e.g. for travis cron job). -The test is skipped if the data wasn't previously fetched and saved to -scikit-learn data folder. +Only 'percent10' mode is tested, as the full data +is too big to use in unit-testing. """ -from sklearn.datasets import fetch_kddcup99 from sklearn.datasets.tests.test_common import check_return_X_y -from sklearn.utils._testing import SkipTest from functools import partial - -def test_percent10(): - try: - data = fetch_kddcup99(download_if_missing=False) - except IOError: - raise SkipTest("kddcup99 dataset can not be loaded.") +def test_percent10(fetch_kddcup99_fxt): + data = fetch_kddcup99_fxt() assert data.data.shape == (494021, 41) assert data.target.shape == (494021,) - data_shuffled = fetch_kddcup99(shuffle=True, random_state=0) + data_shuffled = fetch_kddcup99_fxt(shuffle=True, random_state=0) assert data.data.shape == data_shuffled.data.shape assert data.target.shape == data_shuffled.target.shape - data = fetch_kddcup99('SA') + data = fetch_kddcup99_fxt('SA') assert data.data.shape == (100655, 41) assert data.target.shape == (100655,) - data = fetch_kddcup99('SF') + data = fetch_kddcup99_fxt('SF') assert data.data.shape == (73237, 4) assert data.target.shape == (73237,) - data = fetch_kddcup99('http') + data = fetch_kddcup99_fxt('http') assert data.data.shape == (58725, 3) assert data.target.shape == (58725,) - data = fetch_kddcup99('smtp') + data = fetch_kddcup99_fxt('smtp') assert data.data.shape == (9571, 3) assert data.target.shape == (9571,) - fetch_func = partial(fetch_kddcup99, 'smtp') + fetch_func = partial(fetch_kddcup99_fxt, 'smtp') check_return_X_y(data, fetch_func) -def test_shuffle(): - try: - dataset = fetch_kddcup99(random_state=0, subset='SA', shuffle=True, - percent10=True, download_if_missing=False) - except IOError: - raise SkipTest("kddcup99 dataset can not be loaded.") - +def test_shuffle(fetch_kddcup99_fxt): + dataset = fetch_kddcup99_fxt(random_state=0, subset='SA', shuffle=True, + percent10=True) assert(any(dataset.target[-100:] == b'normal.')) diff --git a/sklearn/datasets/tests/test_lfw.py b/sklearn/datasets/tests/test_lfw.py index 3aa65a68bcdbf..19cda818d8d55 100644 --- a/sklearn/datasets/tests/test_lfw.py +++ b/sklearn/datasets/tests/test_lfw.py @@ -24,10 +24,10 @@ from sklearn.datasets.tests.test_common import check_return_X_y -SCIKIT_LEARN_DATA = tempfile.mkdtemp(prefix="scikit_learn_lfw_test_") -SCIKIT_LEARN_EMPTY_DATA = tempfile.mkdtemp(prefix="scikit_learn_empty_test_") +SCIKIT_LEARN_DATA = None +SCIKIT_LEARN_EMPTY_DATA = None +LFW_HOME = None -LFW_HOME = os.path.join(SCIKIT_LEARN_DATA, 'lfw_home') FAKE_NAMES = [ 'Abdelatif_Smith', 'Abhati_Kepler', @@ -44,6 +44,14 @@ def setup_module(): if not pillow_installed: raise SkipTest("PIL not installed.") + global SCIKIT_LEARN_DATA, SCIKIT_LEARN_EMPTY_DATA, LFW_HOME + + SCIKIT_LEARN_DATA = tempfile.mkdtemp(prefix="scikit_learn_lfw_test_") + LFW_HOME = os.path.join(SCIKIT_LEARN_DATA, 'lfw_home') + + SCIKIT_LEARN_EMPTY_DATA = tempfile.mkdtemp( + prefix="scikit_learn_empty_test_") + if not os.path.exists(LFW_HOME): os.makedirs(LFW_HOME) diff --git a/sklearn/datasets/tests/test_olivetti_faces.py b/sklearn/datasets/tests/test_olivetti_faces.py index 0162676c50af7..f0c7aa1216e76 100644 --- a/sklearn/datasets/tests/test_olivetti_faces.py +++ b/sklearn/datasets/tests/test_olivetti_faces.py @@ -1,28 +1,17 @@ -"""Test Olivetti faces fetcher, if the data is available.""" -import pytest +"""Test Olivetti faces fetcher, if the data is available, +or if specifically requested via environment variable +(e.g. for travis cron job).""" + import numpy as np -from sklearn import datasets from sklearn.utils import Bunch from sklearn.datasets.tests.test_common import check_return_X_y from sklearn.utils._testing import assert_array_equal -def _is_olivetti_faces_not_available(): - try: - datasets.fetch_olivetti_faces(download_if_missing=False) - return False - except IOError: - return True - - -@pytest.mark.skipif( - _is_olivetti_faces_not_available(), - reason='Download Olivetti faces dataset to run this test' -) -def test_olivetti_faces(): - data = datasets.fetch_olivetti_faces(shuffle=True, random_state=0) +def test_olivetti_faces(fetch_olivetti_faces_fxt): + data = fetch_olivetti_faces_fxt(shuffle=True, random_state=0) assert isinstance(data, Bunch) for expected_keys in ('data', 'images', 'target', 'DESCR'): @@ -34,4 +23,4 @@ def test_olivetti_faces(): assert_array_equal(np.unique(np.sort(data.target)), np.arange(40)) # test the return_X_y option - check_return_X_y(data, datasets.fetch_olivetti_faces) + check_return_X_y(data, fetch_olivetti_faces_fxt) diff --git a/sklearn/datasets/tests/test_openml.py b/sklearn/datasets/tests/test_openml.py index cb1a95c27a077..f9969c75d5c8e 100644 --- a/sklearn/datasets/tests/test_openml.py +++ b/sklearn/datasets/tests/test_openml.py @@ -102,7 +102,7 @@ def _fetch_dataset_from_openml(data_id, data_name, data_version, assert data_by_id.target.shape == (expected_observations, len(target_column)) assert data_by_id.target_names == target_column - assert data_by_id.data.dtype == np.float64 + assert data_by_id.data.dtype == expected_data_dtype assert data_by_id.target.dtype == expected_target_dtype assert len(data_by_id.feature_names) == expected_features for feature in data_by_id.feature_names: @@ -118,11 +118,7 @@ def _fetch_dataset_from_openml(data_id, data_name, data_version, if compare_default_target: # check whether the data by id and data by id target are equal data_by_id_default = fetch_openml(data_id=data_id, cache=False) - if data_by_id.data.dtype == np.float64: - np.testing.assert_allclose(data_by_id.data, - data_by_id_default.data) - else: - assert np.array_equal(data_by_id.data, data_by_id_default.data) + np.testing.assert_allclose(data_by_id.data, data_by_id_default.data) if data_by_id.target.dtype == np.float64: np.testing.assert_allclose(data_by_id.target, data_by_id_default.target) @@ -740,7 +736,7 @@ def test_fetch_openml_iris_multitarget(monkeypatch, gzip_response): _fetch_dataset_from_openml(data_id, data_name, data_version, target_column, expected_observations, expected_features, expected_missing, - object, np.float64, expect_sparse=False, + np.float64, np.float64, expect_sparse=False, compare_default_target=False) @@ -759,7 +755,7 @@ def test_fetch_openml_anneal(monkeypatch, gzip_response): _fetch_dataset_from_openml(data_id, data_name, data_version, target_column, expected_observations, expected_features, expected_missing, - object, object, expect_sparse=False, + np.float64, object, expect_sparse=False, compare_default_target=True) @@ -784,7 +780,7 @@ def test_fetch_openml_anneal_multitarget(monkeypatch, gzip_response): _fetch_dataset_from_openml(data_id, data_name, data_version, target_column, expected_observations, expected_features, expected_missing, - object, object, expect_sparse=False, + np.float64, object, expect_sparse=False, compare_default_target=False) @@ -802,7 +798,7 @@ def test_fetch_openml_cpu(monkeypatch, gzip_response): _fetch_dataset_from_openml(data_id, data_name, data_version, target_column, expected_observations, expected_features, expected_missing, - object, np.float64, expect_sparse=False, + np.float64, np.float64, expect_sparse=False, compare_default_target=True) diff --git a/sklearn/datasets/tests/test_rcv1.py b/sklearn/datasets/tests/test_rcv1.py index 7cae454bf158b..2c21201dce40e 100644 --- a/sklearn/datasets/tests/test_rcv1.py +++ b/sklearn/datasets/tests/test_rcv1.py @@ -1,26 +1,17 @@ -"""Test the rcv1 loader. +"""Test the rcv1 loader, if the data is available, +or if specifically requested via environment variable +(e.g. for travis cron job).""" -Skipped if rcv1 is not already downloaded to data_home. -""" - -import errno import scipy.sparse as sp import numpy as np from functools import partial -from sklearn.datasets import fetch_rcv1 from sklearn.datasets.tests.test_common import check_return_X_y from sklearn.utils._testing import assert_almost_equal from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import SkipTest - -def test_fetch_rcv1(): - try: - data1 = fetch_rcv1(shuffle=False, download_if_missing=False) - except IOError as e: - if e.errno == errno.ENOENT: - raise SkipTest("Download RCV1 dataset to run this test.") +def test_fetch_rcv1(fetch_rcv1_fxt): + data1 = fetch_rcv1_fxt(shuffle=False) X1, Y1 = data1.data, data1.target cat_list, s1 = data1.target_names.tolist(), data1.sample_id @@ -48,14 +39,12 @@ def test_fetch_rcv1(): assert num == Y1[:, j].data.size # test shuffling and subset - data2 = fetch_rcv1(shuffle=True, subset='train', random_state=77, - download_if_missing=False) + data2 = fetch_rcv1_fxt(shuffle=True, subset='train', random_state=77) X2, Y2 = data2.data, data2.target s2 = data2.sample_id # test return_X_y option - fetch_func = partial(fetch_rcv1, shuffle=False, subset='train', - download_if_missing=False) + fetch_func = partial(fetch_rcv1_fxt, shuffle=False, subset='train') check_return_X_y(data2, fetch_func) # The first 23149 samples are the training samples diff --git a/sklearn/datasets/tests/test_samples_generator.py b/sklearn/datasets/tests/test_samples_generator.py index e51ca3970bdae..ab712d8c235a6 100644 --- a/sklearn/datasets/tests/test_samples_generator.py +++ b/sklearn/datasets/tests/test_samples_generator.py @@ -222,6 +222,18 @@ def test_make_multilabel_classification_return_indicator_sparse(): assert sp.issparse(Y) +@pytest.mark.parametrize( + "params, err_msg", + [ + ({"n_classes": 0}, "'n_classes' should be an integer"), + ({"length": 0}, "'length' should be an integer") + ] +) +def test_make_multilabel_classification_valid_arguments(params, err_msg): + with pytest.raises(ValueError, match=err_msg): + make_multilabel_classification(**params) + + def test_make_hastie_10_2(): X, y = make_hastie_10_2(n_samples=100, random_state=0) assert X.shape == (100, 10), "X shape mismatch" @@ -312,6 +324,15 @@ def test_make_blobs_n_samples_centers_none(n_samples): "Incorrect number of samples per blob" +def test_make_blobs_return_centers(): + n_samples = [10, 20] + n_features = 3 + X, y, centers = make_blobs(n_samples=n_samples, n_features=n_features, + return_centers=True, random_state=0) + + assert centers.shape == (len(n_samples), n_features) + + def test_make_blobs_error(): n_samples = [20, 20, 20] centers = np.array([[0.0, 0.0], [1.0, 1.0], [0.0, 1.0]]) @@ -476,6 +497,22 @@ def test_make_moons(): err_msg="Point is not on expected unit circle") +def test_make_moons_unbalanced(): + X, y = make_moons(n_samples=(7, 5)) + assert np.sum(y == 0) == 7 and np.sum(y == 1) == 5, \ + 'Number of samples in a moon is wrong' + assert X.shape == (12, 2), "X shape mismatch" + assert y.shape == (12,), "y shape mismatch" + + with pytest.raises(ValueError, match=r'`n_samples` can be either an int ' + r'or a two-element tuple.'): + make_moons(n_samples=[1, 2, 3]) + + with pytest.raises(ValueError, match=r'`n_samples` can be either an int ' + r'or a two-element tuple.'): + make_moons(n_samples=(10,)) + + def test_make_circles(): factor = 0.3 @@ -490,6 +527,7 @@ def test_make_circles(): for x, label in zip(X, y): dist_sqr = ((x - center) ** 2).sum() dist_exp = 1.0 if label == 0 else factor**2 + dist_exp = 1.0 if label == 0 else factor ** 2 assert_almost_equal(dist_sqr, dist_exp, err_msg="Point is not on expected circle") @@ -502,3 +540,20 @@ def test_make_circles(): make_circles(factor=-0.01) with pytest.raises(ValueError): make_circles(factor=1.) + + +def test_make_circles_unbalanced(): + X, y = make_circles(n_samples=(2, 8)) + + assert np.sum(y == 0) == 2, 'Number of samples in inner circle is wrong' + assert np.sum(y == 1) == 8, 'Number of samples in outer circle is wrong' + assert X.shape == (10, 2), "X shape mismatch" + assert y.shape == (10,), "y shape mismatch" + + with pytest.raises(ValueError, match=r'`n_samples` can be either an int ' + r'or a two-element tuple.'): + make_circles(n_samples=[1, 2, 3]) + + with pytest.raises(ValueError, match=r'`n_samples` can be either an int ' + r'or a two-element tuple.'): + make_circles(n_samples=(10,)) diff --git a/sklearn/decomposition/__init__.py b/sklearn/decomposition/__init__.py index 93a51b04f38d2..42f661171eafe 100644 --- a/sklearn/decomposition/__init__.py +++ b/sklearn/decomposition/__init__.py @@ -4,19 +4,33 @@ this module can be regarded as dimensionality reduction techniques. """ -from ._nmf import NMF, non_negative_factorization -from ._pca import PCA -from ._incremental_pca import IncrementalPCA -from ._kernel_pca import KernelPCA -from ._sparse_pca import SparsePCA, MiniBatchSparsePCA -from ._truncated_svd import TruncatedSVD -from ._fastica import FastICA, fastica -from ._dict_learning import (dict_learning, dict_learning_online, +# TODO: remove me in 0.24 (as well as the noqa markers) and +# import the dict_learning func directly from the ._dict_learning +# module instead. +# Pre-cache the import of the deprecated module so that import +# sklearn.decomposition.dict_learning returns the function as in +# 0.21, instead of the module. +# https://github.com/scikit-learn/scikit-learn/issues/15842 +import warnings +with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=FutureWarning) + from .dict_learning import dict_learning + + +from ._nmf import NMF, non_negative_factorization # noqa +from ._pca import PCA # noqa +from ._incremental_pca import IncrementalPCA # noqa +from ._kernel_pca import KernelPCA # noqa +from ._sparse_pca import SparsePCA, MiniBatchSparsePCA # noqa +from ._truncated_svd import TruncatedSVD # noqa +from ._fastica import FastICA, fastica # noqa +from ._dict_learning import (dict_learning_online, sparse_encode, DictionaryLearning, - MiniBatchDictionaryLearning, SparseCoder) -from ._factor_analysis import FactorAnalysis -from ..utils.extmath import randomized_svd -from ._online_lda import LatentDirichletAllocation + MiniBatchDictionaryLearning, SparseCoder) # noqa +from ._factor_analysis import FactorAnalysis # noqa +from ..utils.extmath import randomized_svd # noqa +from ._lda import LatentDirichletAllocation # noqa + __all__ = ['DictionaryLearning', 'FastICA', diff --git a/sklearn/decomposition/_cdnmf_fast.pyx b/sklearn/decomposition/_cdnmf_fast.pyx index b5c8341d92619..9c6b171096ced 100644 --- a/sklearn/decomposition/_cdnmf_fast.pyx +++ b/sklearn/decomposition/_cdnmf_fast.pyx @@ -5,17 +5,18 @@ # Author: Mathieu Blondel, Tom Dupre la Tour # License: BSD 3 clause -cimport cython +from cython cimport floating from libc.math cimport fabs -def _update_cdnmf_fast(double[:, ::1] W, double[:, :] HHt, double[:, :] XHt, - Py_ssize_t[::1] permutation): - cdef double violation = 0 - cdef Py_ssize_t n_components = W.shape[1] - cdef Py_ssize_t n_samples = W.shape[0] # n_features for H update - cdef double grad, pg, hess - cdef Py_ssize_t i, r, s, t +def _update_cdnmf_fast(floating[:, ::1] W, floating[:, :] HHt, + floating[:, :] XHt, Py_ssize_t[::1] permutation): + cdef: + floating violation = 0 + Py_ssize_t n_components = W.shape[1] + Py_ssize_t n_samples = W.shape[0] # n_features for H update + floating grad, pg, hess + Py_ssize_t i, r, s, t with nogil: for s in range(n_components): diff --git a/sklearn/decomposition/_dict_learning.py b/sklearn/decomposition/_dict_learning.py index 0eb69f5b5a74c..9b7ad28f9f235 100644 --- a/sklearn/decomposition/_dict_learning.py +++ b/sklearn/decomposition/_dict_learning.py @@ -361,11 +361,10 @@ def _update_dict(dictionary, Y, code, verbose=False, return_r2=False, Whether to compute and return the residual sum of squares corresponding to the computed solution. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance, default=None + Used for randomly initializing the dictionary. Pass an int for + reproducible results across multiple function calls. + See :term:`Glossary `. positive : boolean, optional Whether to enforce positivity when finding the dictionary. @@ -483,10 +482,9 @@ def dict_learning(X, n_components, alpha, max_iter=100, tol=1e-8, To control the verbosity of the procedure. random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + Used for randomly initializing the dictionary. Pass an int for + reproducible results across multiple function calls. + See :term:`Glossary `. return_n_iter : bool Whether or not to return the number of iterations. @@ -690,10 +688,11 @@ def dict_learning_online(X, n_components=2, alpha=1, n_iter=100, initialization. random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + Used for initializing the dictionary when ``dict_init`` is not + specified, randomly shuffling the data when ``shuffle`` is set to + ``True``, and updating the dictionary. Pass an int for reproducible + results across multiple function calls. + See :term:`Glossary `. return_inner_stats : boolean, optional Return the inner statistics A (dictionary covariance) and B @@ -704,7 +703,7 @@ def dict_learning_online(X, n_components=2, alpha=1, n_iter=100, inner_stats : tuple of (A, B) ndarrays Inner sufficient statistics that are kept by the algorithm. Passing them at initialization is useful in online settings, to - avoid loosing the history of the evolution. + avoid losing the history of the evolution. A (n_components, n_components) is the dictionary covariance matrix. B (n_features, n_components) is the data approximation matrix @@ -952,7 +951,7 @@ class SparseCoder(SparseCodingMixin, BaseEstimator): normalized to unit norm. transform_algorithm : {'lasso_lars', 'lasso_cd', 'lars', 'omp', \ - 'threshold'} + 'threshold'}, default='omp' Algorithm used to transform the data: lars: uses the least angle regression method (linear_model.lars_path) lasso_lars: uses Lars to compute the Lasso solution @@ -963,12 +962,12 @@ class SparseCoder(SparseCodingMixin, BaseEstimator): threshold: squashes to zero all coefficients less than alpha from the projection ``dictionary * X'`` - transform_n_nonzero_coefs : int, ``0.1 * n_features`` by default + transform_n_nonzero_coefs : int, default=0.1*n_features Number of nonzero coefficients to target in each column of the solution. This is only used by `algorithm='lars'` and `algorithm='omp'` and is overridden by `alpha` in the `omp` case. - transform_alpha : float, 1. by default + transform_alpha : float, default=1. If `algorithm='lasso_lars'` or `algorithm='lasso_cd'`, `alpha` is the penalty applied to the L1 norm. If `algorithm='threshold'`, `alpha` is the absolute value of the @@ -977,23 +976,23 @@ class SparseCoder(SparseCodingMixin, BaseEstimator): the reconstruction error targeted. In this case, it overrides `n_nonzero_coefs`. - split_sign : bool, False by default + split_sign : bool, default=False Whether to split the sparse feature vector into the concatenation of its negative part and its positive part. This can improve the performance of downstream classifiers. - n_jobs : int or None, optional (default=None) + n_jobs : int or None, default=None Number of parallel jobs to run. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - positive_code : bool + positive_code : bool, default=False Whether to enforce positivity when finding the code. .. versionadded:: 0.20 - transform_max_iter : int, optional (default=1000) + transform_max_iter : int, default=1000 Maximum number of iterations to perform if `algorithm='lasso_cd'` or `lasso_lars`. @@ -1044,6 +1043,10 @@ def fit(self, X, y=None): """ return self + @property + def n_features_in_(self): + return self.components_.shape[1] + class DictionaryLearning(SparseCodingMixin, BaseEstimator): """Dictionary learning @@ -1132,11 +1135,12 @@ class DictionaryLearning(SparseCodingMixin, BaseEstimator): its negative part and its positive part. This can improve the performance of downstream classifiers. - random_state : int, RandomState instance or None, default=None - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance or None, optional (default=None) + Used for initializing the dictionary when ``dict_init`` is not + specified, randomly shuffling the data when ``shuffle`` is set to + ``True``, and updating the dictionary. Pass an int for reproducible + results across multiple function calls. + See :term:`Glossary `. positive_code : bool, default=False Whether to enforce positivity when finding the code. @@ -1217,7 +1221,7 @@ def fit(self, X, y=None): Returns the object itself """ random_state = check_random_state(self.random_state) - X = check_array(X) + X = self._validate_data(X) if self.n_components is None: n_components = X.shape[1] else: @@ -1323,10 +1327,11 @@ class MiniBatchDictionaryLearning(SparseCodingMixin, BaseEstimator): performance of downstream classifiers. random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + Used for initializing the dictionary when ``dict_init`` is not + specified, randomly shuffling the data when ``shuffle`` is set to + ``True``, and updating the dictionary. Pass an int for reproducible + results across multiple function calls. + See :term:`Glossary `. positive_code : bool Whether to enforce positivity when finding the code. @@ -1351,7 +1356,7 @@ class MiniBatchDictionaryLearning(SparseCodingMixin, BaseEstimator): inner_stats_ : tuple of (A, B) ndarrays Internal sufficient statistics that are kept by the algorithm. - Keeping them is useful in online settings, to avoid loosing the + Keeping them is useful in online settings, to avoid losing the history of the evolution, but they shouldn't have any use for the end user. A (n_components, n_components) is the dictionary covariance matrix. @@ -1423,7 +1428,7 @@ def fit(self, X, y=None): Returns the instance itself. """ random_state = check_random_state(self.random_state) - X = check_array(X) + X = self._validate_data(X) U, (A, B), self.n_iter_ = dict_learning_online( X, self.n_components, self.alpha, @@ -1442,6 +1447,7 @@ def fit(self, X, y=None): # some online fitting (partial_fit) self.inner_stats_ = (A, B) self.iter_offset_ = self.n_iter + self.random_state_ = random_state return self def partial_fit(self, X, y=None, iter_offset=None): diff --git a/sklearn/decomposition/_factor_analysis.py b/sklearn/decomposition/_factor_analysis.py index 14f0648d937bc..7147fd452559c 100644 --- a/sklearn/decomposition/_factor_analysis.py +++ b/sklearn/decomposition/_factor_analysis.py @@ -89,11 +89,10 @@ class FactorAnalysis(TransformerMixin, BaseEstimator): Number of iterations for the power method. 3 by default. Only used if ``svd_method`` equals 'randomized' - random_state : int, RandomState instance or None, optional (default=0) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. Only used when ``svd_method`` equals 'randomized'. + random_state : int, RandomState instance, default=0 + Only used when ``svd_method`` equals 'randomized'. Pass an int for + reproducible results across multiple function calls. + See :term:`Glossary `. Attributes ---------- @@ -169,7 +168,7 @@ def fit(self, X, y=None): ------- self """ - X = check_array(X, copy=self.copy, dtype=np.float64) + X = self._validate_data(X, copy=self.copy, dtype=np.float64) n_samples, n_features = X.shape n_components = self.n_components diff --git a/sklearn/decomposition/_fastica.py b/sklearn/decomposition/_fastica.py index c191f5e41ab41..f9e3a148f6860 100644 --- a/sklearn/decomposition/_fastica.py +++ b/sklearn/decomposition/_fastica.py @@ -202,11 +202,11 @@ def my_g(x): Initial un-mixing array of dimension (n.comp,n.comp). If None (default) then an array of normal r.v.'s is used. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance, default=None + Used to initialize ``w_init`` when not specified, with a + normal distribution. Pass an int, for reproducible results + across multiple function calls. + See :term:`Glossary `. return_X_mean : bool, optional If True, X_mean is returned too. @@ -341,11 +341,11 @@ def my_g(x): w_init : None of an (n_components, n_components) ndarray The mixing matrix to be used to initialize the algorithm. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance, default=None + Used to initialize ``w_init`` when not specified, with a + normal distribution. Pass an int, for reproducible results + across multiple function calls. + See :term:`Glossary `. Attributes ---------- @@ -424,14 +424,12 @@ def _fit(self, X, compute_sources=False): ------- X_new : array-like, shape (n_samples, n_components) """ + + X = self._validate_data(X, copy=self.whiten, dtype=FLOAT_DTYPES, + ensure_min_samples=2).T fun_args = {} if self.fun_args is None else self.fun_args random_state = check_random_state(self.random_state) - # make interface compatible with other decompositions - # a copy is required only for non whitened data - X = check_array(X, copy=self.whiten, dtype=FLOAT_DTYPES, - ensure_min_samples=2).T - alpha = fun_args.get('alpha', 1.0) if not 1 <= alpha <= 2: raise ValueError('alpha must be in [1,2]') diff --git a/sklearn/decomposition/_incremental_pca.py b/sklearn/decomposition/_incremental_pca.py index 9fc0936b880cc..ac535b58e7f5e 100644 --- a/sklearn/decomposition/_incremental_pca.py +++ b/sklearn/decomposition/_incremental_pca.py @@ -104,6 +104,9 @@ class IncrementalPCA(_BasePCA): The number of samples processed by the estimator. Will be reset on new calls to fit, but increments across ``partial_fit`` calls. + batch_size_ : int + Inferred batch size from ``batch_size``. + Examples -------- >>> from sklearn.datasets import load_digits @@ -194,8 +197,8 @@ def fit(self, X, y=None): self.singular_values_ = None self.noise_variance_ = None - X = check_array(X, accept_sparse=['csr', 'csc', 'lil'], - copy=self.copy, dtype=[np.float64, np.float32]) + X = self._validate_data(X, accept_sparse=['csr', 'csc', 'lil'], + copy=self.copy, dtype=[np.float64, np.float32]) n_samples, n_features = X.shape if self.batch_size is None: @@ -270,7 +273,7 @@ def partial_fit(self, X, y=None, check_input=True): self.mean_ = .0 self.var_ = .0 - # Update stats - they are 0 if this is the fisrt step + # Update stats - they are 0 if this is the first step col_mean, col_var, n_total_samples = \ _incremental_mean_and_var( X, last_mean=self.mean_, last_variance=self.var_, diff --git a/sklearn/decomposition/_kernel_pca.py b/sklearn/decomposition/_kernel_pca.py index 1429106495a6e..6f15ebc29f761 100644 --- a/sklearn/decomposition/_kernel_pca.py +++ b/sklearn/decomposition/_kernel_pca.py @@ -9,7 +9,8 @@ from ..utils import check_random_state from ..utils.extmath import svd_flip -from ..utils.validation import check_is_fitted, check_array +from ..utils.validation import (check_is_fitted, check_array, + _check_psd_eigenvalues) from ..exceptions import NotFittedError from ..base import BaseEstimator, TransformerMixin from ..preprocessing import KernelCenterer @@ -75,11 +76,10 @@ class KernelPCA(TransformerMixin, BaseEstimator): When n_components is None, this parameter is ignored and components with zero eigenvalues are removed regardless. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. Used when ``eigen_solver`` == 'arpack'. + random_state : int, RandomState instance, default=None + Used when ``eigen_solver`` == 'arpack'. Pass an int for reproducible + results across multiple function calls. + See :term:`Glossary `. .. versionadded:: 0.18 @@ -211,6 +211,10 @@ def _fit_transform(self, K): maxiter=self.max_iter, v0=v0) + # make sure that the eigenvalues are ok and fix numerical issues + self.lambdas_ = _check_psd_eigenvalues(self.lambdas_, + enable_warnings=False) + # flip eigenvectors' sign to enforce deterministic output self.alphas_, _ = svd_flip(self.alphas_, np.empty_like(self.alphas_).T) @@ -271,7 +275,7 @@ def fit(self, X, y=None): self : object Returns the instance itself. """ - X = check_array(X, accept_sparse='csr', copy=self.copy_X) + X = self._validate_data(X, accept_sparse='csr', copy=self.copy_X) self._centerer = KernelCenterer() K = self._get_kernel(X) self._fit_transform(K) @@ -354,5 +358,6 @@ def inverse_transform(self, X): "the inverse transform is not available.") K = self._get_kernel(X, self.X_transformed_fit_) - + n_samples = self.X_transformed_fit_.shape[0] + K.flat[::n_samples + 1] += self.alpha return np.dot(K, self.dual_coef_) diff --git a/sklearn/decomposition/_online_lda.py b/sklearn/decomposition/_lda.py similarity index 94% rename from sklearn/decomposition/_online_lda.py rename to sklearn/decomposition/_lda.py index c10bad994d9cf..ba68e03a16191 100644 --- a/sklearn/decomposition/_online_lda.py +++ b/sklearn/decomposition/_lda.py @@ -193,7 +193,7 @@ class LatentDirichletAllocation(TransformerMixin, BaseEstimator): evaluate_every : int, optional (default=0) How often to evaluate perplexity. Only used in `fit` method. - set it to 0 or negative number to not evalute perplexity in + set it to 0 or negative number to not evaluate perplexity in training at all. Evaluating perplexity can help you check convergence in training process, but it will also increase total training time. Evaluating perplexity in every iteration might increase training time @@ -222,11 +222,9 @@ class LatentDirichletAllocation(TransformerMixin, BaseEstimator): verbose : int, optional (default=0) Verbosity level. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance, default=None + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary `. Attributes ---------- @@ -469,7 +467,7 @@ def _em_step(self, X, total_samples, batch_update, parallel=None): def _more_tags(self): return {'requires_positive_X': True} - def _check_non_neg_array(self, X, whom): + def _check_non_neg_array(self, X, reset_n_features, whom): """check X format check X format and make sure no negative value in X. @@ -479,7 +477,8 @@ def _check_non_neg_array(self, X, whom): X : array-like or sparse matrix """ - X = check_array(X, accept_sparse='csr') + X = self._validate_data(X, reset=reset_n_features, + accept_sparse='csr') check_non_negative(X, whom) return X @@ -498,13 +497,23 @@ def partial_fit(self, X, y=None): self """ self._check_params() - X = self._check_non_neg_array(X, + first_time = not hasattr(self, 'components_') + + # In theory reset should be equal to `first_time`, but there are tests + # checking the input number of feature and they expect a specific + # string, which is not the same one raised by check_n_features. So we + # don't check n_features_in_ here for now (it's done with adhoc code in + # the estimator anyway). + # TODO: set reset=first_time when addressing reset in + # predict/transform/etc. + reset_n_features = True + X = self._check_non_neg_array(X, reset_n_features, "LatentDirichletAllocation.partial_fit") n_samples, n_features = X.shape batch_size = self.batch_size # initialize parameters or check - if not hasattr(self, 'components_'): + if first_time: self._init_latent_vars(n_features) if n_features != self.components_.shape[1]: @@ -542,7 +551,8 @@ def fit(self, X, y=None): self """ self._check_params() - X = self._check_non_neg_array(X, "LatentDirichletAllocation.fit") + X = self._check_non_neg_array(X, reset_n_features=True, + whom="LatentDirichletAllocation.fit") n_samples, n_features = X.shape max_iter = self.max_iter evaluate_every = self.evaluate_every @@ -611,7 +621,9 @@ def _unnormalized_transform(self, X): check_is_fitted(self) # make sure feature size is the same in fitted model and in X - X = self._check_non_neg_array(X, "LatentDirichletAllocation.transform") + X = self._check_non_neg_array( + X, reset_n_features=True, + whom="LatentDirichletAllocation.transform") n_samples, n_features = X.shape if n_features != self.components_.shape[1]: raise ValueError( @@ -735,7 +747,8 @@ def score(self, X, y=None): score : float Use approximate bound as score. """ - X = self._check_non_neg_array(X, "LatentDirichletAllocation.score") + X = self._check_non_neg_array(X, reset_n_features=True, + whom="LatentDirichletAllocation.score") doc_topic_distr = self._unnormalized_transform(X) score = self._approx_bound(X, doc_topic_distr, sub_sampling=False) @@ -764,8 +777,9 @@ def _perplexity_precomp_distr(self, X, doc_topic_distr=None, """ check_is_fitted(self) - X = self._check_non_neg_array(X, - "LatentDirichletAllocation.perplexity") + X = self._check_non_neg_array( + X, reset_n_features=True, + whom="LatentDirichletAllocation.perplexity") if doc_topic_distr is None: doc_topic_distr = self._unnormalized_transform(X) diff --git a/sklearn/decomposition/_nmf.py b/sklearn/decomposition/_nmf.py index 9d335eb775d8b..86c9acddfea1e 100644 --- a/sklearn/decomposition/_nmf.py +++ b/sklearn/decomposition/_nmf.py @@ -287,11 +287,10 @@ def _initialize_nmf(X, n_components, init=None, eps=1e-6, eps : float Truncate all values less then this in output to zero. - random_state : int, RandomState instance or None, optional, default: None - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. Used when ``random`` == 'nndsvdar' or 'random'. + random_state : int, RandomState instance, default=None + Used when ``init`` == 'nndsvdar' or 'random'. Pass an int for + reproducible results across multiple function calls. + See :term:`Glossary `. Returns ------- @@ -326,18 +325,18 @@ def _initialize_nmf(X, n_components, init=None, eps=1e-6, if init == 'random': avg = np.sqrt(X.mean() / n_components) rng = check_random_state(random_state) - H = avg * rng.randn(n_components, n_features) - W = avg * rng.randn(n_samples, n_components) - # we do not write np.abs(H, out=H) to stay compatible with - # numpy 1.5 and earlier where the 'out' keyword is not - # supported as a kwarg on ufuncs - np.abs(H, H) - np.abs(W, W) + H = avg * rng.randn(n_components, n_features).astype(X.dtype, + copy=False) + W = avg * rng.randn(n_samples, n_components).astype(X.dtype, + copy=False) + np.abs(H, out=H) + np.abs(W, out=W) return W, H # NNDSVD initialization U, S, V = randomized_svd(X, n_components, random_state=random_state) - W, H = np.zeros(U.shape), np.zeros(V.shape) + W = np.zeros_like(U) + H = np.zeros_like(V) # The leading singular triplet is non-negative # so it can be used as is for initialization. @@ -472,11 +471,11 @@ def _fit_coordinate_descent(X, W, H, tol=1e-4, max_iter=200, l1_reg_W=0, shuffle : boolean, default: False If true, randomize the order of coordinates in the CD solver. - random_state : int, RandomState instance or None, optional, default: None - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance, default=None + Used to randomize the coordinates in the CD solver, when + ``shuffle`` is set to ``True``. Pass an int for reproducible + results across multiple function calls. + See :term:`Glossary `. Returns ------- @@ -842,7 +841,7 @@ def _fit_multiplicative_update(X, W, H, beta_loss='frobenius', def non_negative_factorization(X, W=None, H=None, n_components=None, - init='warn', update_H=True, solver='cd', + init=None, update_H=True, solver='cd', beta_loss='frobenius', tol=1e-4, max_iter=200, alpha=0., l1_ratio=0., regularization=None, random_state=None, @@ -891,10 +890,7 @@ def non_negative_factorization(X, W=None, H=None, n_components=None, init : None | 'random' | 'nndsvd' | 'nndsvda' | 'nndsvdar' | 'custom' Method used to initialize the procedure. - Default: 'random'. - - The default value will change from 'random' to None in version 0.23 - to make it consistent with decomposition.NMF. + Default: None. Valid options: @@ -915,6 +911,9 @@ def non_negative_factorization(X, W=None, H=None, n_components=None, - 'custom': use custom matrices W and H + .. versionchanged:: 0.23 + The default value of `init` changed from 'random' to None in 0.23. + update_H : boolean, default: True Set to True, both W and H will be estimated from initial guesses. Set to False, only W will be estimated. @@ -963,11 +962,11 @@ def non_negative_factorization(X, W=None, H=None, n_components=None, Select whether the regularization affects the components (H), the transformation (W), both or none of them. - random_state : int, RandomState instance or None, optional, default: None - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance, default=None + Used for NMF initialisation (when ``init`` == 'nndsvdar' or + 'random'), and in Coordinate Descent. Pass an int for reproducible + results across multiple function calls. + See :term:`Glossary `. verbose : integer, default: 0 The verbosity level. @@ -1004,8 +1003,8 @@ def non_negative_factorization(X, W=None, H=None, n_components=None, Fevotte, C., & Idier, J. (2011). Algorithms for nonnegative matrix factorization with the beta-divergence. Neural Computation, 23(9). """ - - X = check_array(X, accept_sparse=('csr', 'csc'), dtype=float) + X = check_array(X, accept_sparse=('csr', 'csc'), + dtype=[np.float64, np.float32]) check_non_negative(X, "NMF (input X)") beta_loss = _check_string_param(solver, regularization, beta_loss, init) @@ -1028,25 +1027,25 @@ def non_negative_factorization(X, W=None, H=None, n_components=None, raise ValueError("Tolerance for stopping criteria must be " "positive; got (tol=%r)" % tol) - if init == "warn": - if n_components < n_features: - warnings.warn("The default value of init will change from " - "random to None in 0.23 to make it consistent " - "with decomposition.NMF.", FutureWarning) - init = "random" - # check W and H, or initialize them if init == 'custom' and update_H: _check_init(H, (n_components, n_features), "NMF (input H)") _check_init(W, (n_samples, n_components), "NMF (input W)") + if H.dtype != X.dtype or W.dtype != X.dtype: + raise TypeError("H and W should have the same dtype as X. Got " + "H.dtype = {} and W.dtype = {}." + .format(H.dtype, W.dtype)) elif not update_H: _check_init(H, (n_components, n_features), "NMF (input H)") + if H.dtype != X.dtype: + raise TypeError("H should have the same dtype as X. Got H.dtype = " + "{}.".format(H.dtype)) # 'mu' solver should not be initialized by zeros if solver == 'mu': avg = np.sqrt(X.mean() / n_components) - W = np.full((n_samples, n_components), avg) + W = np.full((n_samples, n_components), avg, dtype=X.dtype) else: - W = np.zeros((n_samples, n_components)) + W = np.zeros((n_samples, n_components), dtype=X.dtype) else: W, H = _initialize_nmf(X, n_components, init=init, random_state=random_state) @@ -1163,11 +1162,11 @@ class NMF(TransformerMixin, BaseEstimator): max_iter : integer, default: 200 Maximum number of iterations before timing out. - random_state : int, RandomState instance or None, optional, default: None - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance, default=None + Used for initialisation (when ``init`` == 'nndsvdar' or + 'random'), and in Coordinate Descent. Pass an int for reproducible + results across multiple function calls. + See :term:`Glossary `. alpha : double, default: 0. Constant that multiplies the regularization terms. Set it to zero to @@ -1276,7 +1275,8 @@ def fit_transform(self, X, y=None, W=None, H=None): W : array, shape (n_samples, n_components) Transformed data. """ - X = check_array(X, accept_sparse=('csr', 'csc'), dtype=float) + X = self._validate_data(X, accept_sparse=('csr', 'csc'), + dtype=[np.float64, np.float32]) W, H, n_iter_ = non_negative_factorization( X=X, W=W, H=H, n_components=self.n_components, init=self.init, diff --git a/sklearn/decomposition/_pca.py b/sklearn/decomposition/_pca.py index e3fcf2dfcc6bb..7a0140b01fc9b 100644 --- a/sklearn/decomposition/_pca.py +++ b/sklearn/decomposition/_pca.py @@ -27,7 +27,7 @@ from ..utils.validation import check_is_fitted -def _assess_dimension_(spectrum, rank, n_samples, n_features): +def _assess_dimension(spectrum, rank, n_samples, n_features): """Compute the likelihood of a rank ``rank`` dataset. The dataset is assumed to be embedded in gaussian noise of shape(n, @@ -58,6 +58,8 @@ def _assess_dimension_(spectrum, rank, n_samples, n_features): raise ValueError("The tested rank cannot exceed the rank of the" " dataset") + spectrum_threshold = np.finfo(type(spectrum[0])).eps + pu = -rank * log(2.) for i in range(rank): pu += (gammaln((n_features - i) / 2.) - @@ -67,10 +69,14 @@ def _assess_dimension_(spectrum, rank, n_samples, n_features): pl = -pl * n_samples / 2. if rank == n_features: + # TODO: this line is never executed because _infer_dimension's + # for loop is off by one pv = 0 v = 1 else: v = np.sum(spectrum[rank:]) / (n_features - rank) + if spectrum_threshold > v: + return -np.inf pv = -np.log(v) * n_samples * (n_features - rank) / 2. m = n_features * rank - rank * (rank + 1.) / 2. @@ -80,6 +86,13 @@ def _assess_dimension_(spectrum, rank, n_samples, n_features): spectrum_ = spectrum.copy() spectrum_[rank:n_features] = v for i in range(rank): + if spectrum_[i] < spectrum_threshold: + # TODO: this line is never executed + # (off by one in _infer_dimension) + # this break only happens when rank == n_features and + # spectrum_[i] < spectrum_threshold, otherwise the early return + # above catches this case. + break for j in range(i + 1, len(spectrum)): pa += log((spectrum[i] - spectrum[j]) * (1. / spectrum_[j] - 1. / spectrum_[i])) + log(n_samples) @@ -89,7 +102,7 @@ def _assess_dimension_(spectrum, rank, n_samples, n_features): return ll -def _infer_dimension_(spectrum, n_samples, n_features): +def _infer_dimension(spectrum, n_samples, n_features): """Infers the dimension of a dataset of shape (n_samples, n_features) The dataset is described by its spectrum `spectrum`. @@ -97,7 +110,7 @@ def _infer_dimension_(spectrum, n_samples, n_features): n_spectrum = len(spectrum) ll = np.empty(n_spectrum) for rank in range(n_spectrum): - ll[rank] = _assess_dimension_(spectrum, rank, n_samples, n_features) + ll[rank] = _assess_dimension(spectrum, rank, n_samples, n_features) return ll.argmax() @@ -189,11 +202,10 @@ class PCA(_BasePCA): .. versionadded:: 0.18.0 - random_state : int, RandomState instance or None, optional (default None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. Used when ``svd_solver`` == 'arpack' or 'randomized'. + random_state : int, RandomState instance, default=None + Used when ``svd_solver`` == 'arpack' or 'randomized'. Pass an int + for reproducible results across multiple function calls. + See :term:`Glossary `. .. versionadded:: 0.18.0 @@ -387,8 +399,8 @@ def _fit(self, X): raise TypeError('PCA does not support sparse input. See ' 'TruncatedSVD for a possible alternative.') - X = check_array(X, dtype=[np.float64, np.float32], ensure_2d=True, - copy=self.copy) + X = self._validate_data(X, dtype=[np.float64, np.float32], + ensure_2d=True, copy=self.copy) # Handle n_components==None if self.n_components is None: @@ -459,13 +471,16 @@ def _fit_full(self, X, n_components): # Postprocess the number of components required if n_components == 'mle': n_components = \ - _infer_dimension_(explained_variance_, n_samples, n_features) + _infer_dimension(explained_variance_, n_samples, n_features) elif 0 < n_components < 1.0: # number of components for which the cumulated explained # variance percentage is superior to the desired threshold + # side='right' ensures that number of features selected + # their variance is always greater than n_components float + # passed. More discussion in issue: #15669 ratio_cumsum = stable_cumsum(explained_variance_ratio_) - n_components = np.searchsorted(ratio_cumsum, n_components) + 1 - + n_components = np.searchsorted(ratio_cumsum, n_components, + side='right') + 1 # Compute noise covariance using Probabilistic PCA model # The sigma2 maximum likelihood (cf. eq. 12.46) if n_components < min(n_features, n_samples): diff --git a/sklearn/decomposition/_sparse_pca.py b/sklearn/decomposition/_sparse_pca.py index 3e31994d6894d..158bbefc22e92 100644 --- a/sklearn/decomposition/_sparse_pca.py +++ b/sklearn/decomposition/_sparse_pca.py @@ -79,11 +79,10 @@ class SparsePCA(TransformerMixin, BaseEstimator): verbose : int Controls the verbosity; the higher, the more messages. Defaults to 0. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance, default=None + Used during dictionary learning. Pass an int for reproducible results + across multiple function calls. + See :term:`Glossary `. normalize_components : 'deprecated' This parameter does not have any effect. The components are always @@ -166,7 +165,7 @@ def fit(self, X, y=None): Returns the instance itself. """ random_state = check_random_state(self.random_state) - X = check_array(X) + X = self._validate_data(X) _check_normalize_components( self.normalize_components, self.__class__.__name__ @@ -231,6 +230,14 @@ def transform(self, X): return U + def _more_tags(self): + return { + '_xfail_test': { + "check_methods_subset_invariance": + "fails for the transform method" + } + } + class MiniBatchSparsePCA(SparsePCA): """Mini-batch Sparse Principal Components Analysis @@ -282,11 +289,11 @@ class MiniBatchSparsePCA(SparsePCA): Lasso solution (linear_model.Lasso). Lars will be faster if the estimated components are sparse. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance, default=None + Used for random shuffling when ``shuffle`` is set to ``True``, + during online dictionary learning. Pass an int for reproducible results + across multiple function calls. + See :term:`Glossary `. normalize_components : 'deprecated' This parameter does not have any effect. The components are always @@ -364,7 +371,7 @@ def fit(self, X, y=None): Returns the instance itself. """ random_state = check_random_state(self.random_state) - X = check_array(X) + X = self._validate_data(X) _check_normalize_components( self.normalize_components, self.__class__.__name__ diff --git a/sklearn/decomposition/_truncated_svd.py b/sklearn/decomposition/_truncated_svd.py index 73e4dfbe9f547..940eab56feea8 100644 --- a/sklearn/decomposition/_truncated_svd.py +++ b/sklearn/decomposition/_truncated_svd.py @@ -56,11 +56,10 @@ class TruncatedSVD(TransformerMixin, BaseEstimator): `~sklearn.utils.extmath.randomized_svd` to handle sparse matrices that may have large slowly decaying spectrum. - random_state : int, RandomState instance or None, optional, default = None - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance, default=None + Used during randomized svd. Pass an int for reproducible results across + multiple function calls. + See :term:`Glossary `. tol : float, optional Tolerance for ARPACK. 0 means machine precision. Ignored by randomized @@ -158,8 +157,8 @@ def fit_transform(self, X, y=None): X_new : array, shape (n_samples, n_components) Reduced version of X. This will always be a dense array. """ - X = check_array(X, accept_sparse=['csr', 'csc'], - ensure_min_features=2) + X = self._validate_data(X, accept_sparse=['csr', 'csc'], + ensure_min_features=2) random_state = check_random_state(self.random_state) if self.algorithm == "arpack": diff --git a/sklearn/decomposition/tests/test_dict_learning.py b/sklearn/decomposition/tests/test_dict_learning.py index 9ecc9cbf25598..5f082ffea13ee 100644 --- a/sklearn/decomposition/tests/test_dict_learning.py +++ b/sklearn/decomposition/tests/test_dict_learning.py @@ -498,3 +498,9 @@ def test_sparse_coder_parallel_mmap(): sc = SparseCoder(init_dict, transform_algorithm='omp', n_jobs=2) sc.fit_transform(data) + + +def test_sparse_coder_n_features_in(): + d = np.array([[1, 2, 3], [1, 2, 3]]) + sc = SparseCoder(d) + assert sc.n_features_in_ == d.shape[1] diff --git a/sklearn/decomposition/tests/test_fastica.py b/sklearn/decomposition/tests/test_fastica.py index 40e71f896f638..9f37ac25c2f76 100644 --- a/sklearn/decomposition/tests/test_fastica.py +++ b/sklearn/decomposition/tests/test_fastica.py @@ -158,7 +158,6 @@ def test_fastica_convergence_fail(): s2 = np.ceil(np.sin(np.pi * t)) s = np.c_[s1, s2].T center_and_norm(s) - s1, s2 = s # Mixing matrix mixing = rng.randn(6, 2) @@ -170,7 +169,8 @@ def test_fastica_convergence_fail(): assert_warns(ConvergenceWarning, ica.fit, m.T) -def test_non_square_fastica(add_noise=False): +@pytest.mark.parametrize('add_noise', [True, False]) +def test_non_square_fastica(add_noise): # Test the FastICA algorithm on very simple data. rng = np.random.RandomState(0) diff --git a/sklearn/decomposition/tests/test_kernel_pca.py b/sklearn/decomposition/tests/test_kernel_pca.py index 9b6bd5d4de436..a7a9547bfa33a 100644 --- a/sklearn/decomposition/tests/test_kernel_pca.py +++ b/sklearn/decomposition/tests/test_kernel_pca.py @@ -7,10 +7,12 @@ from sklearn.decomposition import PCA, KernelPCA from sklearn.datasets import make_circles +from sklearn.datasets import make_blobs from sklearn.linear_model import Perceptron from sklearn.pipeline import Pipeline from sklearn.model_selection import GridSearchCV from sklearn.metrics.pairwise import rbf_kernel +from sklearn.utils.validation import _check_psd_eigenvalues def test_kernel_pca(): @@ -214,8 +216,6 @@ def test_kernel_pca_invalid_kernel(): kpca.fit(X_fit) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_gridsearch_pipeline(): # Test if we can do a grid-search to find parameters to separate # circles with a perceptron model. @@ -230,8 +230,6 @@ def test_gridsearch_pipeline(): assert grid_search.best_score_ == 1 -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_gridsearch_pipeline_precomputed(): # Test if we can do a grid-search to find parameters to separate # circles with a perceptron model using a precomputed kernel. @@ -247,8 +245,6 @@ def test_gridsearch_pipeline_precomputed(): assert grid_search.best_score_ == 1 -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_nested_circles(): # Test the linear separability of the first 2D KPCA transform X, y = make_circles(n_samples=400, factor=.3, noise=.05, @@ -270,3 +266,32 @@ def test_nested_circles(): # The data is perfectly linearly separable in that space train_score = Perceptron(max_iter=5).fit(X_kpca, y).score(X_kpca, y) assert train_score == 1.0 + + +def test_kernel_conditioning(): + """ Test that ``_check_psd_eigenvalues`` is correctly called + Non-regression test for issue #12140 (PR #12145)""" + + # create a pathological X leading to small non-zero eigenvalue + X = [[5, 1], + [5+1e-8, 1e-8], + [5+1e-8, 0]] + kpca = KernelPCA(kernel="linear", n_components=2, + fit_inverse_transform=True) + kpca.fit(X) + + # check that the small non-zero eigenvalue was correctly set to zero + assert kpca.lambdas_.min() == 0 + assert np.all(kpca.lambdas_ == _check_psd_eigenvalues(kpca.lambdas_)) + + +@pytest.mark.parametrize("kernel", + ["linear", "poly", "rbf", "sigmoid", "cosine"]) +def test_kernel_pca_inverse_transform(kernel): + X, *_ = make_blobs(n_samples=100, n_features=4, centers=[[1, 1, 1, 1]], + random_state=0) + + kp = KernelPCA(n_components=2, kernel=kernel, fit_inverse_transform=True) + X_trans = kp.fit_transform(X) + X_inv = kp.inverse_transform(X_trans) + assert_allclose(X, X_inv) diff --git a/sklearn/decomposition/tests/test_nmf.py b/sklearn/decomposition/tests/test_nmf.py index d98ad551513e7..c81a0136177dc 100644 --- a/sklearn/decomposition/tests/test_nmf.py +++ b/sklearn/decomposition/tests/test_nmf.py @@ -1,6 +1,5 @@ import numpy as np import scipy.sparse as sp -import numbers from scipy import linalg from sklearn.decomposition import NMF, non_negative_factorization @@ -10,10 +9,10 @@ import pytest from sklearn.utils._testing import assert_raise_message -from sklearn.utils._testing import assert_warns_message from sklearn.utils._testing import assert_array_equal from sklearn.utils._testing import assert_array_almost_equal from sklearn.utils._testing import assert_almost_equal +from sklearn.utils._testing import assert_allclose from sklearn.utils._testing import ignore_warnings from sklearn.utils.extmath import squared_norm from sklearn.base import clone @@ -224,10 +223,6 @@ def test_non_negative_factorization_checking(): A = np.ones((2, 2)) # Test parameters checking is public function nnmf = non_negative_factorization - msg = ("The default value of init will change from " - "random to None in 0.23 to make it consistent " - "with decomposition.NMF.") - assert_warns_message(FutureWarning, msg, nnmf, A, A, A, np.int64(1)) msg = ("Number of components must be a positive integer; " "got (n_components=1.5)") assert_raise_message(ValueError, msg, nnmf, A, A, A, 1.5, 'random') @@ -250,11 +245,6 @@ def _beta_divergence_dense(X, W, H, beta): Used as a reference for testing nmf._beta_divergence. """ - if isinstance(X, numbers.Number): - W = np.array([[W]]) - H = np.array([[H]]) - X = np.array([[X]]) - WH = np.dot(W, H) if beta == 2: @@ -508,3 +498,48 @@ def test_nmf_underflow(): X[0, 0] = 1e-323 res = nmf._beta_divergence(X, W, H, beta=1.0) assert_almost_equal(res, ref) + + +@pytest.mark.parametrize("dtype_in, dtype_out", [ + (np.float32, np.float32), + (np.float64, np.float64), + (np.int32, np.float64), + (np.int64, np.float64)]) +@pytest.mark.parametrize("solver", ["cd", "mu"]) +def test_nmf_dtype_match(dtype_in, dtype_out, solver): + # Check that NMF preserves dtype (float32 and float64) + X = np.random.RandomState(0).randn(20, 15).astype(dtype_in, copy=False) + np.abs(X, out=X) + nmf = NMF(solver=solver) + + assert nmf.fit(X).transform(X).dtype == dtype_out + assert nmf.fit_transform(X).dtype == dtype_out + assert nmf.components_.dtype == dtype_out + + +@pytest.mark.parametrize("solver", ["cd", "mu"]) +def test_nmf_float32_float64_consistency(solver): + # Check that the result of NMF is the same between float32 and float64 + X = np.random.RandomState(0).randn(50, 7) + np.abs(X, out=X) + nmf32 = NMF(solver=solver, random_state=0) + W32 = nmf32.fit_transform(X.astype(np.float32)) + nmf64 = NMF(solver=solver, random_state=0) + W64 = nmf64.fit_transform(X) + + assert_allclose(W32, W64, rtol=1e-6, atol=1e-5) + + +def test_nmf_custom_init_dtype_error(): + # Check that an error is raise if custom H and/or W don't have the same + # dtype as X. + rng = np.random.RandomState(0) + X = rng.random_sample((20, 15)) + H = rng.random_sample((15, 15)).astype(np.float32) + W = rng.random_sample((20, 15)) + + with pytest.raises(TypeError, match="should have the same dtype as X"): + NMF(init='custom').fit(X, H=H, W=W) + + with pytest.raises(TypeError, match="should have the same dtype as X"): + non_negative_factorization(X, H=H, update_H=False) diff --git a/sklearn/decomposition/tests/test_online_lda.py b/sklearn/decomposition/tests/test_online_lda.py index fdf993f2759bf..ca8392616e761 100644 --- a/sklearn/decomposition/tests/test_online_lda.py +++ b/sklearn/decomposition/tests/test_online_lda.py @@ -8,8 +8,8 @@ import pytest from sklearn.decomposition import LatentDirichletAllocation -from sklearn.decomposition._online_lda import (_dirichlet_expectation_1d, - _dirichlet_expectation_2d) +from sklearn.decomposition._lda import (_dirichlet_expectation_1d, + _dirichlet_expectation_2d) from sklearn.utils._testing import assert_allclose from sklearn.utils._testing import assert_array_almost_equal diff --git a/sklearn/decomposition/tests/test_pca.py b/sklearn/decomposition/tests/test_pca.py index 826a8cc082c3a..438478a55f6fa 100644 --- a/sklearn/decomposition/tests/test_pca.py +++ b/sklearn/decomposition/tests/test_pca.py @@ -7,8 +7,9 @@ from sklearn import datasets from sklearn.decomposition import PCA -from sklearn.decomposition._pca import _assess_dimension_ -from sklearn.decomposition._pca import _infer_dimension_ +from sklearn.datasets import load_iris +from sklearn.decomposition._pca import _assess_dimension +from sklearn.decomposition._pca import _infer_dimension iris = datasets.load_iris() PCA_SOLVERS = ['full', 'arpack', 'randomized', 'auto'] @@ -332,7 +333,7 @@ def test_infer_dim_1(): pca = PCA(n_components=p, svd_solver='full') pca.fit(X) spect = pca.explained_variance_ - ll = np.array([_assess_dimension_(spect, k, n, p) for k in range(p)]) + ll = np.array([_assess_dimension(spect, k, n, p) for k in range(p)]) assert ll[1] > ll.max() - .01 * n @@ -347,7 +348,7 @@ def test_infer_dim_2(): pca = PCA(n_components=p, svd_solver='full') pca.fit(X) spect = pca.explained_variance_ - assert _infer_dimension_(spect, n, p) > 1 + assert _infer_dimension(spect, n, p) > 1 def test_infer_dim_3(): @@ -360,7 +361,7 @@ def test_infer_dim_3(): pca = PCA(n_components=p, svd_solver='full') pca.fit(X) spect = pca.explained_variance_ - assert _infer_dimension_(spect, n, p) > 2 + assert _infer_dimension(spect, n, p) > 2 @pytest.mark.parametrize( @@ -532,7 +533,10 @@ def check_pca_float_dtype_preservation(svd_solver): assert pca_64.transform(X_64).dtype == np.float64 assert pca_32.transform(X_32).dtype == np.float32 - assert_allclose(pca_64.components_, pca_32.components_, rtol=1e-4) + # the rtol is set such that the test passes on all platforms tested on + # conda-forge: PR#15775 + # see: https://github.com/conda-forge/scikit-learn-feedstock/pull/113 + assert_allclose(pca_64.components_, pca_32.components_, rtol=2e-4) def check_pca_int_dtype_upcast_to_double(svd_solver): @@ -552,3 +556,70 @@ def check_pca_int_dtype_upcast_to_double(svd_solver): assert pca_32.transform(X_i32).dtype == np.float64 assert_allclose(pca_64.components_, pca_32.components_, rtol=1e-4) + + +def test_pca_n_components_mostly_explained_variance_ratio(): + # when n_components is the second highest cumulative sum of the + # explained_variance_ratio_, then n_components_ should equal the + # number of features in the dataset #15669 + X, y = load_iris(return_X_y=True) + pca1 = PCA().fit(X, y) + + n_components = pca1.explained_variance_ratio_.cumsum()[-2] + pca2 = PCA(n_components=n_components).fit(X, y) + assert pca2.n_components_ == X.shape[1] + + +def test_infer_dim_bad_spec(): + # Test a spectrum that drops to near zero for PR #16224 + spectrum = np.array([1, 1e-30, 1e-30, 1e-30]) + n_samples = 10 + n_features = 5 + ret = _infer_dimension(spectrum, n_samples, n_features) + assert ret == 0 + + +def test_assess_dimension_error_rank_greater_than_features(): + # Test error when tested rank is greater than the number of features + # for PR #16224 + spectrum = np.array([1, 1e-30, 1e-30, 1e-30]) + n_samples = 10 + n_features = 4 + rank = 5 + with pytest.raises(ValueError, match="The tested rank cannot exceed " + "the rank of the dataset"): + _assess_dimension(spectrum, rank, n_samples, n_features) + + +def test_assess_dimension_small_eigenvalues(): + # Test tiny eigenvalues appropriately when using 'mle' + # for PR #16224 + spectrum = np.array([1, 1e-30, 1e-30, 1e-30]) + n_samples = 10 + n_features = 5 + rank = 3 + ret = _assess_dimension(spectrum, rank, n_samples, n_features) + assert ret == -np.inf + + +def test_infer_dim_mle(): + # Test small eigenvalues when 'mle' with pathological 'X' dataset + # for PR #16224 + X, _ = datasets.make_classification(n_informative=1, n_repeated=18, + n_redundant=1, n_clusters_per_class=1, + random_state=42) + pca = PCA(n_components='mle').fit(X) + assert pca.n_components_ == 0 + + +def test_fit_mle_too_few_samples(): + # Tests that an error is raised when the number of samples is smaller + # than the number of features during an mle fit for PR #16224 + X, _ = datasets.make_classification(n_samples=20, n_features=21, + random_state=42) + + pca = PCA(n_components='mle', svd_solver='full') + with pytest.raises(ValueError, match="n_components='mle' is only " + "supported if " + "n_samples >= n_features"): + pca.fit(X) diff --git a/sklearn/discriminant_analysis.py b/sklearn/discriminant_analysis.py index 4492d0868994d..2bd3948f2e013 100644 --- a/sklearn/discriminant_analysis.py +++ b/sklearn/discriminant_analysis.py @@ -11,7 +11,6 @@ import warnings import numpy as np -from .exceptions import ChangedBehaviorWarning from scipy import linalg from scipy.special import expit @@ -24,6 +23,7 @@ from .utils.multiclass import check_classification_targets from .utils.extmath import softmax from .preprocessing import StandardScaler +from .utils.validation import _deprecate_positional_args __all__ = ['LinearDiscriminantAnalysis', 'QuadraticDiscriminantAnalysis'] @@ -34,10 +34,10 @@ def _cov(X, shrinkage=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Input data. - shrinkage : string or float, optional + shrinkage : {'empirical', 'auto'} or float, default=None Shrinkage parameter, possible values: - None or 'empirical': no shrinkage (default). - 'auto': automatic shrinkage using the Ledoit-Wolf lemma. @@ -45,7 +45,7 @@ def _cov(X, shrinkage=None): Returns ------- - s : array, shape (n_features, n_features) + s : ndarray of shape (n_features, n_features) Estimated covariance matrix. """ shrinkage = "empirical" if shrinkage is None else shrinkage @@ -74,15 +74,15 @@ def _class_means(X, y): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Input data. - y : array-like, shape (n_samples,) or (n_samples, n_targets) + y : array-like of shape (n_samples,) or (n_samples, n_targets) Target values. Returns ------- - means : array-like, shape (n_classes, n_features) + means : array-like of shape (n_classes, n_features) Class means. """ classes, y = np.unique(y, return_inverse=True) @@ -98,16 +98,16 @@ def _class_cov(X, y, priors, shrinkage=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Input data. - y : array-like, shape (n_samples,) or (n_samples, n_targets) + y : array-like of shape (n_samples,) or (n_samples, n_targets) Target values. - priors : array-like, shape (n_classes,) + priors : array-like of shape (n_classes,) Class priors. - shrinkage : string or float, optional + shrinkage : 'auto' or float, default=None Shrinkage parameter, possible values: - None: no shrinkage (default). - 'auto': automatic shrinkage using the Ledoit-Wolf lemma. @@ -115,7 +115,7 @@ def _class_cov(X, y, priors, shrinkage=None): Returns ------- - cov : array-like, shape (n_features, n_features) + cov : array-like of shape (n_features, n_features) Class covariance matrix. """ classes = np.unique(y) @@ -146,7 +146,7 @@ class LinearDiscriminantAnalysis(BaseEstimator, LinearClassifierMixin, Parameters ---------- - solver : string, optional + solver : {'svd', 'lsqr', 'eigen'}, default='svd' Solver to use, possible values: - 'svd': Singular value decomposition (default). Does not compute the covariance matrix, therefore this solver is @@ -154,7 +154,7 @@ class LinearDiscriminantAnalysis(BaseEstimator, LinearClassifierMixin, - 'lsqr': Least squares solution, can be combined with shrinkage. - 'eigen': Eigenvalue decomposition, can be combined with shrinkage. - shrinkage : string or float, optional + shrinkage : 'auto' or float, default=None Shrinkage parameter, possible values: - None: no shrinkage (default). - 'auto': automatic shrinkage using the Ledoit-Wolf lemma. @@ -162,55 +162,56 @@ class LinearDiscriminantAnalysis(BaseEstimator, LinearClassifierMixin, Note that shrinkage works only with 'lsqr' and 'eigen' solvers. - priors : array, optional, shape (n_classes,) + priors : array-like of shape (n_classes,), default=None Class priors. - n_components : int, optional (default=None) + n_components : int, default=None Number of components (<= min(n_classes - 1, n_features)) for dimensionality reduction. If None, will be set to min(n_classes - 1, n_features). - store_covariance : bool, optional + store_covariance : bool, default=False Additionally compute class covariance matrix (default False), used only in 'svd' solver. .. versionadded:: 0.17 - tol : float, optional, (default 1.0e-4) + tol : float, default=1.0e-4 Threshold used for rank estimation in SVD solver. .. versionadded:: 0.17 Attributes ---------- - coef_ : array, shape (n_features,) or (n_classes, n_features) + coef_ : ndarray of shape (n_features,) or (n_classes, n_features) Weight vector(s). - intercept_ : array, shape (n_classes,) + intercept_ : ndarray of shape (n_classes,) Intercept term. - covariance_ : array-like, shape (n_features, n_features) - Covariance matrix (shared by all classes). + covariance_ : array-like of shape (n_features, n_features) + Covariance matrix (shared by all classes). Only available + `store_covariance` is True. - explained_variance_ratio_ : array, shape (n_components,) + explained_variance_ratio_ : ndarray of shape (n_components,) Percentage of variance explained by each of the selected components. If ``n_components`` is not set then all components are stored and the sum of explained variances is equal to 1.0. Only available when eigen or svd solver is used. - means_ : array-like, shape (n_classes, n_features) + means_ : array-like of shape (n_classes, n_features) Class means. - priors_ : array-like, shape (n_classes,) + priors_ : array-like of shape (n_classes,) Class priors (sum to 1). - scalings_ : array-like, shape (rank, n_classes - 1) + scalings_ : array-like of shape (rank, n_classes - 1) Scaling of the features in the space spanned by the class centroids. - xbar_ : array-like, shape (n_features,) + xbar_ : array-like of shape (n_features,) Overall mean. - classes_ : array-like, shape (n_classes,) + classes_ : array-like of shape (n_classes,) Unique class labels. See also @@ -246,8 +247,8 @@ class LinearDiscriminantAnalysis(BaseEstimator, LinearClassifierMixin, >>> print(clf.predict([[-0.8, -1]])) [1] """ - - def __init__(self, solver='svd', shrinkage=None, priors=None, + @_deprecate_positional_args + def __init__(self, *, solver='svd', shrinkage=None, priors=None, n_components=None, store_covariance=False, tol=1e-4): self.solver = solver self.shrinkage = shrinkage @@ -267,15 +268,15 @@ def _solve_lsqr(self, X, y, shrinkage): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data. - y : array-like, shape (n_samples,) or (n_samples, n_classes) + y : array-like of shape (n_samples,) or (n_samples, n_classes) Target values. - shrinkage : string or float, optional + shrinkage : 'auto', float or None Shrinkage parameter, possible values: - - None: no shrinkage (default). + - None: no shrinkage. - 'auto': automatic shrinkage using the Ledoit-Wolf lemma. - float between 0 and 1: fixed shrinkage parameter. @@ -305,15 +306,15 @@ class scatter). This solver supports both classification and Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data. - y : array-like, shape (n_samples,) or (n_samples, n_targets) + y : array-like of shape (n_samples,) or (n_samples, n_targets) Target values. - shrinkage : string or float, optional + shrinkage : 'auto', float or None Shrinkage parameter, possible values: - - None: no shrinkage (default). + - None: no shrinkage. - 'auto': automatic shrinkage using the Ledoit-Wolf lemma. - float between 0 and 1: fixed shrinkage constant. @@ -349,10 +350,10 @@ def _solve_svd(self, X, y): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data. - y : array-like, shape (n_samples,) or (n_samples, n_targets) + y : array-like of shape (n_samples,) or (n_samples, n_targets) Target values. """ n_samples, n_features = X.shape @@ -417,15 +418,14 @@ def fit(self, X, y): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data. - y : array, shape (n_samples,) + y : array-like of shape (n_samples,) Target values. """ - # FIXME: Future warning to be removed in 0.23 - X, y = check_X_y(X, y, ensure_min_samples=2, estimator=self, - dtype=[np.float64, np.float32]) + X, y = self._validate_data(X, y, ensure_min_samples=2, estimator=self, + dtype=[np.float64, np.float32]) self.classes_ = unique_labels(y) n_samples, _ = X.shape n_classes = len(self.classes_) @@ -455,21 +455,11 @@ def fit(self, X, y): self._max_components = max_components else: if self.n_components > max_components: - warnings.warn( + raise ValueError( "n_components cannot be larger than min(n_features, " - "n_classes - 1). Using min(n_features, " - "n_classes - 1) = min(%d, %d - 1) = %d components." - % (X.shape[1], len(self.classes_), max_components), - ChangedBehaviorWarning) - future_msg = ("In version 0.23, setting n_components > min(" - "n_features, n_classes - 1) will raise a " - "ValueError. You should set n_components to None" - " (default), or a value smaller or equal to " - "min(n_features, n_classes - 1).") - warnings.warn(future_msg, FutureWarning) - self._max_components = max_components - else: - self._max_components = self.n_components + "n_classes - 1)." + ) + self._max_components = self.n_components if self.solver == 'svd': if self.shrinkage is not None: @@ -494,12 +484,12 @@ def transform(self, X): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Input data. Returns ------- - X_new : array, shape (n_samples, n_components) + X_new : ndarray of shape (n_samples, n_components) Transformed data. """ if self.solver == 'lsqr': @@ -520,12 +510,12 @@ def predict_proba(self, X): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Input data. Returns ------- - C : array, shape (n_samples, n_classes) + C : ndarray of shape (n_samples, n_classes) Estimated probabilities. """ check_is_fitted(self) @@ -542,12 +532,12 @@ def predict_log_proba(self, X): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Input data. Returns ------- - C : array, shape (n_samples, n_classes) + C : ndarray of shape (n_samples, n_classes) Estimated log probabilities. """ return np.log(self.predict_proba(X)) @@ -569,20 +559,20 @@ class QuadraticDiscriminantAnalysis(ClassifierMixin, BaseEstimator): Parameters ---------- - priors : array, optional, shape = [n_classes] + priors : ndarray of shape (n_classes,), default=None Priors on classes - reg_param : float, optional + reg_param : float, default=0.0 Regularizes the covariance estimate as ``(1-reg_param)*Sigma + reg_param*np.eye(n_features)`` - store_covariance : boolean + store_covariance : bool, default=False If True the covariance matrices are computed and stored in the `self.covariance_` attribute. .. versionadded:: 0.17 - tol : float, optional, default 1.0e-4 + tol : float, default=1.0e-4 Threshold used for rank estimation. .. versionadded:: 0.17 @@ -590,26 +580,27 @@ class QuadraticDiscriminantAnalysis(ClassifierMixin, BaseEstimator): Attributes ---------- covariance_ : list of array-like of shape (n_features, n_features) - Covariance matrices of each class. + Covariance matrices of each class. Only available + `store_covariance` is True. means_ : array-like of shape (n_classes, n_features) Class means. - priors_ : array-like of shape (n_classes) + priors_ : array-like of shape (n_classes,) Class priors (sum to 1). - rotations_ : list of arrays - For each class k an array of shape [n_features, n_k], with + rotations_ : list of ndarrays + For each class k an array of shape (n_features, n_k), with ``n_k = min(n_features, number of elements in class k)`` It is the rotation of the Gaussian distribution, i.e. its principal axis. - scalings_ : list of arrays - For each class k an array of shape [n_k]. It contains the scaling + scalings_ : list of ndarrays + For each class k an array of shape (n_k,). It contains the scaling of the Gaussian distributions along its principal axes, i.e. the variance in the rotated coordinate system. - classes_ : array-like, shape (n_classes,) + classes_ : array-like of shape (n_classes,) Unique class labels. Examples @@ -629,8 +620,8 @@ class QuadraticDiscriminantAnalysis(ClassifierMixin, BaseEstimator): sklearn.discriminant_analysis.LinearDiscriminantAnalysis: Linear Discriminant Analysis """ - - def __init__(self, priors=None, reg_param=0., store_covariance=False, + @_deprecate_positional_args + def __init__(self, *, priors=None, reg_param=0., store_covariance=False, tol=1.0e-4): self.priors = np.asarray(priors) if priors is not None else None self.reg_param = reg_param @@ -653,10 +644,10 @@ def fit(self, X, y): Training vector, where n_samples is the number of samples and n_features is the number of features. - y : array, shape = [n_samples] + y : array-like of shape (n_samples,) Target values (integers) """ - X, y = check_X_y(X, y) + X, y = self._validate_data(X, y) check_classification_targets(y) self.classes_, y = np.unique(y, return_inverse=True) n_samples, n_features = X.shape diff --git a/sklearn/dummy.py b/sklearn/dummy.py index 04322f0fc3bd1..daa2c1ff0da11 100644 --- a/sklearn/dummy.py +++ b/sklearn/dummy.py @@ -18,7 +18,7 @@ from .utils.stats import _weighted_percentile from .utils.multiclass import class_distribution from .utils import deprecated - +from .utils.validation import _deprecate_positional_args class DummyClassifier(MultiOutputMixin, ClassifierMixin, BaseEstimator): """ @@ -50,17 +50,17 @@ class DummyClassifier(MultiOutputMixin, ClassifierMixin, BaseEstimator): .. versionchanged:: 0.22 The default value of `strategy` will change to "prior" in version 0.24. Starting from version 0.22, a warning will be raised if - `strategy` is not explicity set. + `strategy` is not explicitly set. .. versionadded:: 0.17 Dummy Classifier now supports prior fitting strategy using parameter *prior*. random_state : int, RandomState instance or None, optional, default=None - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + Controls the randomness to generate the predictions when + ``strategy='stratified'`` or ``strategy='uniform'``. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. constant : int or str or array-like of shape (n_outputs,) The explicit constant as predicted by the "constant" strategy. This @@ -98,8 +98,8 @@ class DummyClassifier(MultiOutputMixin, ClassifierMixin, BaseEstimator): >>> dummy_clf.score(X, y) 0.75 """ - - def __init__(self, strategy="warn", random_state=None, + @_deprecate_positional_args + def __init__(self, *, strategy="warn", random_state=None, constant=None): self.strategy = strategy self.random_state = random_state @@ -156,7 +156,9 @@ def fit(self, X, y, sample_weight=None): self.n_outputs_ = y.shape[1] - check_consistent_length(X, y, sample_weight) + self.n_features_in_ = None # No input validation is done for X + + check_consistent_length(X, y) if sample_weight is not None: sample_weight = _check_sample_weight(sample_weight, X) @@ -245,7 +247,7 @@ def predict(self, X): classes_ = [np.array([c]) for c in constant] y = _random_choice_csc(n_samples, classes_, class_prob, - self.random_state) + self.random_state) else: if self._strategy in ("most_frequent", "prior"): y = np.tile([classes_[k][class_prior_[k].argmax()] for @@ -354,7 +356,13 @@ def predict_log_proba(self, X): return [np.log(p) for p in proba] def _more_tags(self): - return {'poor_score': True, 'no_validation': True} + return { + 'poor_score': True, 'no_validation': True, + '_xfail_test': { + 'check_methods_subset_invariance': + 'fails for the predict method' + } + } def score(self, X, y, sample_weight=None): """Returns the mean accuracy on the given test data and labels. @@ -453,8 +461,8 @@ class DummyRegressor(MultiOutputMixin, RegressorMixin, BaseEstimator): >>> dummy_regr.score(X, y) 0.0 """ - - def __init__(self, strategy="mean", constant=None, quantile=None): + @_deprecate_positional_args + def __init__(self, *, strategy="mean", constant=None, quantile=None): self.strategy = strategy self.constant = constant self.quantile = quantile @@ -483,6 +491,7 @@ def fit(self, X, y, sample_weight=None): % (self.strategy, allowed_strategies)) y = check_array(y, ensure_2d=False) + self.n_features_in_ = None # No input validation is done for X if len(y) == 0: raise ValueError("y must not be empty.") diff --git a/sklearn/ensemble/__init__.py b/sklearn/ensemble/__init__.py index ae3f98db5cbf1..27acb2fbcf00a 100644 --- a/sklearn/ensemble/__init__.py +++ b/sklearn/ensemble/__init__.py @@ -21,7 +21,6 @@ from ._stacking import StackingClassifier from ._stacking import StackingRegressor -from . import partial_dependence __all__ = ["BaseEnsemble", "RandomForestClassifier", "RandomForestRegressor", @@ -31,4 +30,4 @@ "GradientBoostingRegressor", "AdaBoostClassifier", "AdaBoostRegressor", "VotingClassifier", "VotingRegressor", "StackingClassifier", "StackingRegressor", - "partial_dependence"] + ] diff --git a/sklearn/ensemble/_bagging.py b/sklearn/ensemble/_bagging.py index 2a9ed512113d8..d73f38954d21a 100644 --- a/sklearn/ensemble/_bagging.py +++ b/sklearn/ensemble/_bagging.py @@ -82,7 +82,7 @@ def _parallel_build_estimators(n_estimators, ensemble, X, y, sample_weight, print("Building estimator %d of %d for this parallel run " "(total %d)..." % (i + 1, n_estimators, total_n_estimators)) - random_state = np.random.RandomState(seeds[i]) + random_state = seeds[i] estimator = ensemble._make_estimator(append=False, random_state=random_state) @@ -259,10 +259,10 @@ def _fit(self, X, y, max_samples=None, max_depth=None, sample_weight=None): The target values (class labels in classification, real numbers in regression). - max_samples : int or float, optional (default=None) + max_samples : int or float, default=None Argument to use instead of self.max_samples. - max_depth : int, optional (default=None) + max_depth : int, default=None Override value used when constructing base estimator. Only supported if the base estimator has a max_depth parameter. @@ -278,9 +278,9 @@ def _fit(self, X, y, max_samples=None, max_depth=None, sample_weight=None): random_state = check_random_state(self.random_state) # Convert data (X is required to be 2d and indexable) - X, y = check_X_y( - X, y, ['csr', 'csc'], dtype=None, force_all_finite=False, - multi_output=True + X, y = self._validate_data( + X, y, accept_sparse=['csr', 'csc'], dtype=None, + force_all_finite=False, multi_output=True ) if sample_weight is not None: sample_weight = _check_sample_weight(sample_weight, X, dtype=None) @@ -405,9 +405,8 @@ def _get_estimators_indices(self): for seed in self._seeds: # Operations accessing random_state must be performed identically # to those in `_parallel_build_estimators()` - random_state = np.random.RandomState(seed) feature_indices, sample_indices = _generate_bagging_indices( - random_state, self.bootstrap_features, self.bootstrap, + seed, self.bootstrap_features, self.bootstrap, self.n_features_, self._n_samples, self._max_features, self._max_samples) @@ -456,37 +455,40 @@ class BaggingClassifier(ClassifierMixin, BaseBagging): Parameters ---------- - base_estimator : object or None, optional (default=None) + base_estimator : object, default=None The base estimator to fit on random subsets of the dataset. If None, then the base estimator is a decision tree. - n_estimators : int, optional (default=10) + n_estimators : int, default=10 The number of base estimators in the ensemble. - max_samples : int or float, optional (default=1.0) - The number of samples to draw from X to train each base estimator. + max_samples : int or float, default=1.0 + The number of samples to draw from X to train each base estimator (with + replacement by default, see `bootstrap` for more details). - If int, then draw `max_samples` samples. - If float, then draw `max_samples * X.shape[0]` samples. - max_features : int or float, optional (default=1.0) - The number of features to draw from X to train each base estimator. + max_features : int or float, default=1.0 + The number of features to draw from X to train each base estimator ( + without replacement by default, see `bootstrap_features` for more + details). - If int, then draw `max_features` features. - If float, then draw `max_features * X.shape[1]` features. - bootstrap : boolean, optional (default=True) + bootstrap : bool, default=True Whether samples are drawn with replacement. If False, sampling without replacement is performed. - bootstrap_features : boolean, optional (default=False) + bootstrap_features : bool, default=False Whether features are drawn with replacement. - oob_score : bool, optional (default=False) + oob_score : bool, default=False Whether to use out-of-bag samples to estimate the generalization error. - warm_start : bool, optional (default=False) + warm_start : bool, default=False When set to True, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just fit a whole new ensemble. See :term:`the Glossary `. @@ -494,19 +496,21 @@ class BaggingClassifier(ClassifierMixin, BaseBagging): .. versionadded:: 0.17 *warm_start* constructor parameter. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of jobs to run in parallel for both :meth:`fit` and :meth:`predict`. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState, default=None + Controls the random resampling of the original dataset + (sample wise and feature wise). + If the base estimator accepts a `random_state` attribute, a different + seed is generated for each instance in the ensemble. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. - verbose : int, optional (default=0) + verbose : int, default=0 Controls the verbosity when fitting and predicting. Attributes @@ -527,7 +531,7 @@ class BaggingClassifier(ClassifierMixin, BaseBagging): estimators_features_ : list of arrays The subset of drawn features for each base estimator. - classes_ : array of shape (n_classes,) + classes_ : ndarray of shape (n_classes,) The classes labels. n_classes_ : int or list @@ -537,7 +541,7 @@ class BaggingClassifier(ClassifierMixin, BaseBagging): Score of the training dataset obtained using an out-of-bag estimate. This attribute exists only when ``oob_score`` is True. - oob_decision_function_ : array of shape (n_samples, n_classes) + oob_decision_function_ : ndarray of shape (n_samples, n_classes) Decision function computed with out-of-bag estimate on the training set. If n_estimators is small it might be possible that a data point was never left out during the bootstrap. In this case, @@ -689,7 +693,7 @@ def predict_proba(self, X): Returns ------- - p : array of shape (n_samples, n_classes) + p : ndarray of shape (n_samples, n_classes) The class probabilities of the input samples. The order of the classes corresponds to that in the attribute :term:`classes_`. """ @@ -739,7 +743,7 @@ def predict_log_proba(self, X): Returns ------- - p : array of shape (n_samples, n_classes) + p : ndarray of shape (n_samples, n_classes) The class log-probabilities of the input samples. The order of the classes corresponds to that in the attribute :term:`classes_`. """ @@ -794,7 +798,7 @@ def decision_function(self, X): Returns ------- - score : array, shape = [n_samples, k] + score : ndarray of shape (n_samples, k) The decision function of the input samples. The columns correspond to the classes in sorted order, as they appear in the attribute ``classes_``. Regression and binary classification are special @@ -858,54 +862,59 @@ class BaggingRegressor(RegressorMixin, BaseBagging): Parameters ---------- - base_estimator : object or None, optional (default=None) + base_estimator : object, default=None The base estimator to fit on random subsets of the dataset. If None, then the base estimator is a decision tree. - n_estimators : int, optional (default=10) + n_estimators : int, default=10 The number of base estimators in the ensemble. - max_samples : int or float, optional (default=1.0) - The number of samples to draw from X to train each base estimator. + max_samples : int or float, default=1.0 + The number of samples to draw from X to train each base estimator (with + replacement by default, see `bootstrap` for more details). - If int, then draw `max_samples` samples. - If float, then draw `max_samples * X.shape[0]` samples. - max_features : int or float, optional (default=1.0) - The number of features to draw from X to train each base estimator. + max_features : int or float, default=1.0 + The number of features to draw from X to train each base estimator ( + without replacement by default, see `bootstrap_features` for more + details). - If int, then draw `max_features` features. - If float, then draw `max_features * X.shape[1]` features. - bootstrap : boolean, optional (default=True) + bootstrap : bool, default=True Whether samples are drawn with replacement. If False, sampling without replacement is performed. - bootstrap_features : boolean, optional (default=False) + bootstrap_features : bool, default=False Whether features are drawn with replacement. - oob_score : bool + oob_score : bool, default=False Whether to use out-of-bag samples to estimate the generalization error. - warm_start : bool, optional (default=False) + warm_start : bool, default=False When set to True, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just fit a whole new ensemble. See :term:`the Glossary `. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of jobs to run in parallel for both :meth:`fit` and :meth:`predict`. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState, default=None + Controls the random resampling of the original dataset + (sample wise and feature wise). + If the base estimator accepts a `random_state` attribute, a different + seed is generated for each instance in the ensemble. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. - verbose : int, optional (default=0) + verbose : int, default=0 Controls the verbosity when fitting and predicting. Attributes diff --git a/sklearn/ensemble/_base.py b/sklearn/ensemble/_base.py index 7f9036785b1a3..23db107874c9b 100644 --- a/sklearn/ensemble/_base.py +++ b/sklearn/ensemble/_base.py @@ -1,6 +1,4 @@ -""" -Base class for ensemble-based estimators. -""" +"""Base class for ensemble-based estimators.""" # Authors: Gilles Louppe # License: BSD 3 clause @@ -17,18 +15,18 @@ from ..base import is_classifier, is_regressor from ..base import BaseEstimator from ..base import MetaEstimatorMixin -from ..utils import Bunch +from ..utils import Bunch, _print_elapsed_time from ..utils import check_random_state from ..utils.metaestimators import _BaseComposition -MAX_RAND_SEED = np.iinfo(np.int32).max - -def _parallel_fit_estimator(estimator, X, y, sample_weight=None): +def _fit_single_estimator(estimator, X, y, sample_weight=None, + message_clsname=None, message=None): """Private function used to fit an estimator within a job.""" if sample_weight is not None: try: - estimator.fit(X, y, sample_weight=sample_weight) + with _print_elapsed_time(message_clsname, message): + estimator.fit(X, y, sample_weight=sample_weight) except TypeError as exc: if "unexpected keyword argument 'sample_weight'" in str(exc): raise TypeError( @@ -37,24 +35,24 @@ def _parallel_fit_estimator(estimator, X, y, sample_weight=None): ) from exc raise else: - estimator.fit(X, y) + with _print_elapsed_time(message_clsname, message): + estimator.fit(X, y) return estimator def _set_random_states(estimator, random_state=None): - """Sets fixed random_state parameters for an estimator + """Set fixed random_state parameters for an estimator. Finds all parameters ending ``random_state`` and sets them to integers derived from ``random_state``. Parameters ---------- - estimator : estimator supporting get/set_params Estimator with potential randomness managed by random_state parameters. - random_state : int, RandomState instance or None, optional (default=None) + random_state : int or RandomState, default=None If int, random_state is the seed used by the random number generator; If RandomState instance, random_state is the random number generator; If None, the random number generator is the RandomState instance used @@ -74,7 +72,7 @@ def _set_random_states(estimator, random_state=None): to_set = {} for key in sorted(estimator.get_params(deep=True)): if key == 'random_state' or key.endswith('__random_state'): - to_set[key] = random_state.randint(MAX_RAND_SEED) + to_set[key] = random_state.randint(np.iinfo(np.int32).max) if to_set: estimator.set_params(**to_set) @@ -88,13 +86,13 @@ class BaseEnsemble(MetaEstimatorMixin, BaseEstimator, metaclass=ABCMeta): Parameters ---------- - base_estimator : object, optional (default=None) + base_estimator : object The base estimator from which the ensemble is built. - n_estimators : integer + n_estimators : int, default=10 The number of estimators in the ensemble. - estimator_params : list of strings + estimator_params : list of str, default=tuple() The list of attributes to use as parameters when instantiating a new base estimator. If none are given, default parameters are used. @@ -106,6 +104,7 @@ class BaseEnsemble(MetaEstimatorMixin, BaseEstimator, metaclass=ABCMeta): estimators_ : list of estimators The collection of fitted base estimators. """ + # overwrite _required_parameters from MetaEstimatorMixin _required_parameters = [] @@ -122,8 +121,10 @@ def __init__(self, base_estimator, n_estimators=10, # self.estimators_ needs to be filled by the derived classes in fit. def _validate_estimator(self, default=None): - """Check the estimator and the n_estimator attribute, set the - `base_estimator_` attribute.""" + """Check the estimator and the n_estimator attribute. + + Sets the base_estimator_` attributes. + """ if not isinstance(self.n_estimators, numbers.Integral): raise ValueError("n_estimators must be an integer, " "got {0}.".format(type(self.n_estimators))) @@ -159,15 +160,15 @@ def _make_estimator(self, append=True, random_state=None): return estimator def __len__(self): - """Returns the number of estimators in the ensemble.""" + """Return the number of estimators in the ensemble.""" return len(self.estimators_) def __getitem__(self, index): - """Returns the index'th estimator in the ensemble.""" + """Return the index'th estimator in the ensemble.""" return self.estimators_[index] def __iter__(self): - """Returns iterator over estimators in the ensemble.""" + """Return iterator over estimators in the ensemble.""" return iter(self.estimators_) @@ -204,6 +205,7 @@ class _BaseHeterogeneousEnsemble(MetaEstimatorMixin, _BaseComposition, training data. If an estimator has been set to `'drop'`, it will not appear in `estimators_`. """ + _required_parameters = ['estimators'] @property @@ -277,7 +279,7 @@ def get_params(self, deep=True): Parameters ---------- - deep : bool + deep : bool, default=True Setting it to True gets the various classifiers and the parameters of the classifiers as well. """ diff --git a/sklearn/ensemble/_forest.py b/sklearn/ensemble/_forest.py index 2dd600dc8f984..d6784b10f05d3 100644 --- a/sklearn/ensemble/_forest.py +++ b/sklearn/ensemble/_forest.py @@ -210,14 +210,14 @@ def apply(self, X): Parameters ---------- - X : {array-like or sparse matrix} of shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, its dtype will be converted to ``dtype=np.float32``. If a sparse matrix is provided, it will be converted into a sparse ``csr_matrix``. Returns ------- - X_leaves : array_like, shape = [n_samples, n_estimators] + X_leaves : ndarray of shape (n_samples, n_estimators) For each datapoint x in X and for each tree in the forest, return the index of the leaf x ends up in. """ @@ -237,18 +237,19 @@ def decision_path(self, X): Parameters ---------- - X : {array-like or sparse matrix} of shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, its dtype will be converted to ``dtype=np.float32``. If a sparse matrix is provided, it will be converted into a sparse ``csr_matrix``. Returns ------- - indicator : sparse csr array, shape = [n_samples, n_nodes] - Return a node indicator matrix where non zero elements - indicates that the samples goes through the nodes. + indicator : sparse matrix of shape (n_samples, n_nodes) + Return a node indicator matrix where non zero elements indicates + that the samples goes through the nodes. The matrix is of CSR + format. - n_nodes_ptr : array of size (n_estimators + 1, ) + n_nodes_ptr : ndarray of shape (n_estimators + 1,) The columns from indicator[n_nodes_ptr[i]:n_nodes_ptr[i+1]] gives the indicator value for the i-th estimator. @@ -271,7 +272,7 @@ def fit(self, X, y, sample_weight=None): Parameters ---------- - X : array-like or sparse matrix of shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The training input samples. Internally, its dtype will be converted to ``dtype=np.float32``. If a sparse matrix is provided, it will be converted into a sparse ``csc_matrix``. @@ -292,10 +293,15 @@ def fit(self, X, y, sample_weight=None): self : object """ # Validate or convert input data - X = check_array(X, accept_sparse="csc", dtype=DTYPE) - y = check_array(y, accept_sparse='csc', ensure_2d=False, dtype=None) + if issparse(y): + raise ValueError( + "sparse multilabel-indicator for y is not supported." + ) + X, y = self._validate_data(X, y, multi_output=True, + accept_sparse="csc", dtype=DTYPE) if sample_weight is not None: sample_weight = _check_sample_weight(sample_weight, X) + if issparse(X): # Pre-sort indices to avoid that each individual tree of the # ensemble sorts the indices. @@ -414,12 +420,20 @@ def _validate_X_predict(self, X): @property def feature_importances_(self): """ - Return the feature importances (the higher, the more important the - feature). + The impurity-based feature importances. + + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. Returns ------- - feature_importances_ : array, shape = [n_features] + feature_importances_ : ndarray of shape (n_features,) The values of this array sum to 1, unless all trees are single node trees consisting of only the root node, in which case it will be an array of zeros. @@ -599,14 +613,14 @@ def predict(self, X): Parameters ---------- - X : array-like or sparse matrix of shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, its dtype will be converted to ``dtype=np.float32``. If a sparse matrix is provided, it will be converted into a sparse ``csr_matrix``. Returns ------- - y : array-like of shape (n_samples,) or (n_samples, n_outputs) + y : ndarray of shape (n_samples,) or (n_samples, n_outputs) The predicted classes. """ proba = self.predict_proba(X) @@ -639,14 +653,14 @@ def predict_proba(self, X): Parameters ---------- - X : array-like or sparse matrix of shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, its dtype will be converted to ``dtype=np.float32``. If a sparse matrix is provided, it will be converted into a sparse ``csr_matrix``. Returns ------- - p : array of shape (n_samples, n_classes), or a list of n_outputs + p : ndarray of shape (n_samples, n_classes), or a list of n_outputs such arrays if n_outputs > 1. The class probabilities of the input samples. The order of the classes corresponds to that in the attribute :term:`classes_`. @@ -686,14 +700,14 @@ def predict_log_proba(self, X): Parameters ---------- - X : array-like or sparse matrix of shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, its dtype will be converted to ``dtype=np.float32``. If a sparse matrix is provided, it will be converted into a sparse ``csr_matrix``. Returns ------- - p : array of shape (n_samples, n_classes), or a list of n_outputs + p : ndarray of shape (n_samples, n_classes), or a list of n_outputs such arrays if n_outputs > 1. The class probabilities of the input samples. The order of the classes corresponds to that in the attribute :term:`classes_`. @@ -751,14 +765,14 @@ def predict(self, X): Parameters ---------- - X : array-like or sparse matrix of shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, its dtype will be converted to ``dtype=np.float32``. If a sparse matrix is provided, it will be converted into a sparse ``csr_matrix``. Returns ------- - y : array-like of shape (n_samples,) or (n_samples, n_outputs) + y : ndarray of shape (n_samples,) or (n_samples, n_outputs) The predicted values. """ check_is_fitted(self) @@ -832,6 +846,36 @@ def _set_oob_score(self, X, y): self.oob_score_ /= self.n_outputs_ + def _compute_partial_dependence_recursion(self, grid, target_features): + """Fast partial dependence computation. + + Parameters + ---------- + grid : ndarray of shape (n_samples, n_target_features) + The grid points on which the partial dependence should be + evaluated. + target_features : ndarray of shape (n_target_features) + The set of target features for which the partial dependence + should be evaluated. + + Returns + ------- + averaged_predictions : ndarray of shape (n_samples,) + The value of the partial dependence function on each grid point. + """ + grid = np.asarray(grid, dtype=DTYPE, order='C') + averaged_predictions = np.zeros(shape=grid.shape[0], + dtype=np.float64, order='C') + + for tree in self.estimators_: + # Note: we don't sum in parallel because the GIL isn't released in + # the fast method. + tree.tree_.compute_partial_dependence( + grid, target_features, averaged_predictions) + # Average over the forest + averaged_predictions /= len(self.estimators_) + + return averaged_predictions class RandomForestClassifier(ForestClassifier): """ @@ -848,24 +892,24 @@ class RandomForestClassifier(ForestClassifier): Parameters ---------- - n_estimators : integer, optional (default=100) + n_estimators : int, default=100 The number of trees in the forest. .. versionchanged:: 0.22 The default value of ``n_estimators`` changed from 10 to 100 in 0.22. - criterion : string, optional (default="gini") + criterion : {"gini", "entropy"}, default="gini" The function to measure the quality of a split. Supported criteria are "gini" for the Gini impurity and "entropy" for the information gain. Note: this parameter is tree-specific. - max_depth : integer or None, optional (default=None) + max_depth : int, default=None The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples. - min_samples_split : int, float, optional (default=2) + min_samples_split : int or float, default=2 The minimum number of samples required to split an internal node: - If int, then consider `min_samples_split` as the minimum number. @@ -876,7 +920,7 @@ class RandomForestClassifier(ForestClassifier): .. versionchanged:: 0.18 Added float values for fractions. - min_samples_leaf : int, float, optional (default=1) + min_samples_leaf : int or float, default=1 The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least ``min_samples_leaf`` training samples in each of the left and @@ -891,12 +935,12 @@ class RandomForestClassifier(ForestClassifier): .. versionchanged:: 0.18 Added float values for fractions. - min_weight_fraction_leaf : float, optional (default=0.) + min_weight_fraction_leaf : float, default=0.0 The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. - max_features : int, float, string or None, optional (default="auto") + max_features : {"auto", "sqrt", "log2"}, int or float, default="auto" The number of features to consider when looking for the best split: - If int, then consider `max_features` features at each split. @@ -912,12 +956,12 @@ class RandomForestClassifier(ForestClassifier): valid partition of the node samples is found, even if it requires to effectively inspect more than ``max_features`` features. - max_leaf_nodes : int or None, optional (default=None) + max_leaf_nodes : int, default=None Grow trees with ``max_leaf_nodes`` in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. - min_impurity_decrease : float, optional (default=0.) + min_impurity_decrease : float, default=0.0 A node will be split if this split induces a decrease of the impurity greater than or equal to this value. @@ -935,48 +979,49 @@ class RandomForestClassifier(ForestClassifier): .. versionadded:: 0.19 - min_impurity_split : float, (default=1e-7) + min_impurity_split : float, default=None Threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf. .. deprecated:: 0.19 ``min_impurity_split`` has been deprecated in favor of ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` will change from 1e-7 to 0 in 0.23 and it + ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it will be removed in 0.25. Use ``min_impurity_decrease`` instead. - bootstrap : boolean, optional (default=True) + bootstrap : bool, default=True Whether bootstrap samples are used when building trees. If False, the - whole datset is used to build each tree. + whole dataset is used to build each tree. - oob_score : bool (default=False) + oob_score : bool, default=False Whether to use out-of-bag samples to estimate the generalization accuracy. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of jobs to run in parallel. :meth:`fit`, :meth:`predict`, :meth:`decision_path` and :meth:`apply` are all parallelized over the trees. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState, default=None + Controls both the randomness of the bootstrapping of the samples used + when building trees (if ``bootstrap=True``) and the sampling of the + features to consider when looking for the best split at each node + (if ``max_features < n_features``). + See :term:`Glossary ` for details. - verbose : int, optional (default=0) + verbose : int, default=0 Controls the verbosity when fitting and predicting. - warm_start : bool, optional (default=False) + warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just fit a whole new forest. See :term:`the Glossary `. - class_weight : dict, list of dicts, "balanced", "balanced_subsample" or \ - None, optional (default=None) + class_weight : {"balanced", "balanced_subsample"}, dict or list of dicts, \ + default=None Weights associated with classes in the form ``{class_label: weight}``. If not given, all classes are supposed to have weight one. For multi-output problems, a list of dicts can be provided in the same @@ -1001,7 +1046,7 @@ class RandomForestClassifier(ForestClassifier): Note that these weights will be multiplied with sample_weight (passed through the fit method) if sample_weight is specified. - ccp_alpha : non-negative float, optional (default=0.0) + ccp_alpha : non-negative float, default=0.0 Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ``ccp_alpha`` will be chosen. By default, no pruning is performed. See @@ -1029,7 +1074,7 @@ class RandomForestClassifier(ForestClassifier): estimators_ : list of DecisionTreeClassifier The collection of fitted sub-estimators. - classes_ : array of shape (n_classes,) or a list of such arrays + classes_ : ndarray of shape (n_classes,) or a list of such arrays The classes labels (single output problem), or a list of arrays of class labels (multi-output problem). @@ -1044,13 +1089,21 @@ class labels (multi-output problem). The number of outputs when ``fit`` is performed. feature_importances_ : ndarray of shape (n_features,) - The feature importances (the higher, the more important the feature). + The impurity-based feature importances. + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. oob_score_ : float Score of the training dataset obtained using an out-of-bag estimate. This attribute exists only when ``oob_score`` is True. - oob_decision_function_ : array of shape (n_samples, n_classes) + oob_decision_function_ : ndarray of shape (n_samples, n_classes) Decision function computed with out-of-bag estimate on the training set. If n_estimators is small it might be possible that a data point was never left out during the bootstrap. In this case, @@ -1068,8 +1121,6 @@ class labels (multi-output problem). >>> clf = RandomForestClassifier(max_depth=2, random_state=0) >>> clf.fit(X, y) RandomForestClassifier(max_depth=2, random_state=0) - >>> print(clf.feature_importances_) - [0.14205973 0.76664038 0.0282433 0.06305659] >>> print(clf.predict([[0, 0, 0, 0]])) [1] @@ -1161,14 +1212,14 @@ class RandomForestRegressor(ForestRegressor): Parameters ---------- - n_estimators : integer, optional (default=10) + n_estimators : int, default=100 The number of trees in the forest. .. versionchanged:: 0.22 The default value of ``n_estimators`` changed from 10 to 100 in 0.22. - criterion : string, optional (default="mse") + criterion : {"mse", "mae"}, default="mse" The function to measure the quality of a split. Supported criteria are "mse" for the mean squared error, which is equal to variance reduction as feature selection criterion, and "mae" for the mean @@ -1177,12 +1228,12 @@ class RandomForestRegressor(ForestRegressor): .. versionadded:: 0.18 Mean Absolute Error (MAE) criterion. - max_depth : integer or None, optional (default=None) + max_depth : int, default=None The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples. - min_samples_split : int, float, optional (default=2) + min_samples_split : int or float, default=2 The minimum number of samples required to split an internal node: - If int, then consider `min_samples_split` as the minimum number. @@ -1193,7 +1244,7 @@ class RandomForestRegressor(ForestRegressor): .. versionchanged:: 0.18 Added float values for fractions. - min_samples_leaf : int, float, optional (default=1) + min_samples_leaf : int or float, default=1 The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least ``min_samples_leaf`` training samples in each of the left and @@ -1208,12 +1259,12 @@ class RandomForestRegressor(ForestRegressor): .. versionchanged:: 0.18 Added float values for fractions. - min_weight_fraction_leaf : float, optional (default=0.) + min_weight_fraction_leaf : float, default=0.0 The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. - max_features : int, float, string or None, optional (default="auto") + max_features : {"auto", "sqrt", "log2"}, int or float, default="auto" The number of features to consider when looking for the best split: - If int, then consider `max_features` features at each split. @@ -1229,12 +1280,12 @@ class RandomForestRegressor(ForestRegressor): valid partition of the node samples is found, even if it requires to effectively inspect more than ``max_features`` features. - max_leaf_nodes : int or None, optional (default=None) + max_leaf_nodes : int, default=None Grow trees with ``max_leaf_nodes`` in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. - min_impurity_decrease : float, optional (default=0.) + min_impurity_decrease : float, default=0.0 A node will be split if this split induces a decrease of the impurity greater than or equal to this value. @@ -1252,46 +1303,47 @@ class RandomForestRegressor(ForestRegressor): .. versionadded:: 0.19 - min_impurity_split : float, (default=1e-7) + min_impurity_split : float, default=None Threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf. .. deprecated:: 0.19 ``min_impurity_split`` has been deprecated in favor of ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` will change from 1e-7 to 0 in 0.23 and it + ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it will be removed in 0.25. Use ``min_impurity_decrease`` instead. - bootstrap : boolean, optional (default=True) + bootstrap : bool, default=True Whether bootstrap samples are used when building trees. If False, the - whole datset is used to build each tree. + whole dataset is used to build each tree. - oob_score : bool, optional (default=False) + oob_score : bool, default=False whether to use out-of-bag samples to estimate the R^2 on unseen data. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of jobs to run in parallel. :meth:`fit`, :meth:`predict`, :meth:`decision_path` and :meth:`apply` are all parallelized over the trees. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState, default=None + Controls both the randomness of the bootstrapping of the samples used + when building trees (if ``bootstrap=True``) and the sampling of the + features to consider when looking for the best split at each node + (if ``max_features < n_features``). + See :term:`Glossary ` for details. - verbose : int, optional (default=0) + verbose : int, default=0 Controls the verbosity when fitting and predicting. - warm_start : bool, optional (default=False) + warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just fit a whole new forest. See :term:`the Glossary `. - ccp_alpha : non-negative float, optional (default=0.0) + ccp_alpha : non-negative float, default=0.0 Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ``ccp_alpha`` will be chosen. By default, no pruning is performed. See @@ -1320,7 +1372,15 @@ class RandomForestRegressor(ForestRegressor): The collection of fitted sub-estimators. feature_importances_ : ndarray of shape (n_features,) - The feature importances (the higher, the more important the feature). + The impurity-based feature importances. + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. n_features_ : int The number of features when ``fit`` is performed. @@ -1346,8 +1406,6 @@ class RandomForestRegressor(ForestRegressor): >>> regr = RandomForestRegressor(max_depth=2, random_state=0) >>> regr.fit(X, y) RandomForestRegressor(max_depth=2, random_state=0) - >>> print(regr.feature_importances_) - [0.18146984 0.81473937 0.00145312 0.00233767] >>> print(regr.predict([[0, 0, 0, 0]])) [-8.32987858] @@ -1442,23 +1500,23 @@ class ExtraTreesClassifier(ForestClassifier): Parameters ---------- - n_estimators : integer, optional (default=10) + n_estimators : int, default=100 The number of trees in the forest. .. versionchanged:: 0.22 The default value of ``n_estimators`` changed from 10 to 100 in 0.22. - criterion : string, optional (default="gini") + criterion : {"gini", "entropy"}, default="gini" The function to measure the quality of a split. Supported criteria are "gini" for the Gini impurity and "entropy" for the information gain. - max_depth : integer or None, optional (default=None) + max_depth : int, default=None The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples. - min_samples_split : int, float, optional (default=2) + min_samples_split : int or float, default=2 The minimum number of samples required to split an internal node: - If int, then consider `min_samples_split` as the minimum number. @@ -1469,7 +1527,7 @@ class ExtraTreesClassifier(ForestClassifier): .. versionchanged:: 0.18 Added float values for fractions. - min_samples_leaf : int, float, optional (default=1) + min_samples_leaf : int or float, default=1 The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least ``min_samples_leaf`` training samples in each of the left and @@ -1484,12 +1542,12 @@ class ExtraTreesClassifier(ForestClassifier): .. versionchanged:: 0.18 Added float values for fractions. - min_weight_fraction_leaf : float, optional (default=0.) + min_weight_fraction_leaf : float, default=0.0 The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. - max_features : int, float, string or None, optional (default="auto") + max_features : {"auto", "sqrt", "log2"}, int or float, default="auto" The number of features to consider when looking for the best split: - If int, then consider `max_features` features at each split. @@ -1505,12 +1563,12 @@ class ExtraTreesClassifier(ForestClassifier): valid partition of the node samples is found, even if it requires to effectively inspect more than ``max_features`` features. - max_leaf_nodes : int or None, optional (default=None) + max_leaf_nodes : int, default=None Grow trees with ``max_leaf_nodes`` in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. - min_impurity_decrease : float, optional (default=0.) + min_impurity_decrease : float, default=0.0 A node will be split if this split induces a decrease of the impurity greater than or equal to this value. @@ -1528,47 +1586,52 @@ class ExtraTreesClassifier(ForestClassifier): .. versionadded:: 0.19 - min_impurity_split : float, (default=1e-7) + min_impurity_split : float, default=None Threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf. .. deprecated:: 0.19 ``min_impurity_split`` has been deprecated in favor of ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` will change from 1e-7 to 0 in 0.23 and it + ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it will be removed in 0.25. Use ``min_impurity_decrease`` instead. - bootstrap : boolean, optional (default=False) + bootstrap : bool, default=False Whether bootstrap samples are used when building trees. If False, the - whole datset is used to build each tree. + whole dataset is used to build each tree. - oob_score : bool, optional (default=False) + oob_score : bool, default=False Whether to use out-of-bag samples to estimate the generalization accuracy. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of jobs to run in parallel. :meth:`fit`, :meth:`predict`, :meth:`decision_path` and :meth:`apply` are all parallelized over the trees. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState, default=None + Controls 3 sources of randomness: + + - the bootstrapping of the samples used when building trees + (if ``bootstrap=True``) + - the sampling of the features to consider when looking for the best + split at each node (if ``max_features < n_features``) + - the draw of the splits for each of the `max_features` - verbose : int, optional (default=0) + See :term:`Glossary ` for details. + + verbose : int, default=0 Controls the verbosity when fitting and predicting. - warm_start : bool, optional (default=False) + warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just fit a whole new forest. See :term:`the Glossary `. - class_weight : dict, list of dicts, "balanced", "balanced_subsample" or \ - None, optional (default=None) + class_weight : {"balanced", "balanced_subsample"}, dict or list of dicts, \ + default=None Weights associated with classes in the form ``{class_label: weight}``. If not given, all classes are supposed to have weight one. For multi-output problems, a list of dicts can be provided in the same @@ -1593,7 +1656,7 @@ class ExtraTreesClassifier(ForestClassifier): Note that these weights will be multiplied with sample_weight (passed through the fit method) if sample_weight is specified. - ccp_alpha : non-negative float, optional (default=0.0) + ccp_alpha : non-negative float, default=0.0 Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ``ccp_alpha`` will be chosen. By default, no pruning is performed. See @@ -1614,14 +1677,14 @@ class ExtraTreesClassifier(ForestClassifier): Attributes ---------- - base_estimator_ : ExtraTreeClassifier + base_estimator_ : ExtraTreesClassifier The child estimator template used to create the collection of fitted sub-estimators. estimators_ : list of DecisionTreeClassifier The collection of fitted sub-estimators. - classes_ : array of shape (n_classes,) or a list of such arrays + classes_ : ndarray of shape (n_classes,) or a list of such arrays The classes labels (single output problem), or a list of arrays of class labels (multi-output problem). @@ -1630,7 +1693,15 @@ class labels (multi-output problem). number of classes for each output (multi-output problem). feature_importances_ : ndarray of shape (n_features,) - The feature importances (the higher, the more important the feature). + The impurity-based feature importances. + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. n_features_ : int The number of features when ``fit`` is performed. @@ -1642,7 +1713,7 @@ class labels (multi-output problem). Score of the training dataset obtained using an out-of-bag estimate. This attribute exists only when ``oob_score`` is True. - oob_decision_function_ : array of shape (n_samples, n_classes) + oob_decision_function_ : ndarray of shape (n_samples, n_classes) Decision function computed with out-of-bag estimate on the training set. If n_estimators is small it might be possible that a data point was never left out during the bootstrap. In this case, @@ -1742,14 +1813,14 @@ class ExtraTreesRegressor(ForestRegressor): Parameters ---------- - n_estimators : integer, optional (default=10) + n_estimators : int, default=100 The number of trees in the forest. .. versionchanged:: 0.22 The default value of ``n_estimators`` changed from 10 to 100 in 0.22. - criterion : string, optional (default="mse") + criterion : {"mse", "mae"}, default="mse" The function to measure the quality of a split. Supported criteria are "mse" for the mean squared error, which is equal to variance reduction as feature selection criterion, and "mae" for the mean @@ -1758,12 +1829,12 @@ class ExtraTreesRegressor(ForestRegressor): .. versionadded:: 0.18 Mean Absolute Error (MAE) criterion. - max_depth : integer or None, optional (default=None) + max_depth : int, default=None The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples. - min_samples_split : int, float, optional (default=2) + min_samples_split : int or float, default=2 The minimum number of samples required to split an internal node: - If int, then consider `min_samples_split` as the minimum number. @@ -1774,7 +1845,7 @@ class ExtraTreesRegressor(ForestRegressor): .. versionchanged:: 0.18 Added float values for fractions. - min_samples_leaf : int, float, optional (default=1) + min_samples_leaf : int or float, default=1 The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least ``min_samples_leaf`` training samples in each of the left and @@ -1789,12 +1860,12 @@ class ExtraTreesRegressor(ForestRegressor): .. versionchanged:: 0.18 Added float values for fractions. - min_weight_fraction_leaf : float, optional (default=0.) + min_weight_fraction_leaf : float, default=0.0 The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. - max_features : int, float, string or None, optional (default="auto") + max_features : {"auto", "sqrt", "log2"} int or float, default="auto" The number of features to consider when looking for the best split: - If int, then consider `max_features` features at each split. @@ -1810,12 +1881,12 @@ class ExtraTreesRegressor(ForestRegressor): valid partition of the node samples is found, even if it requires to effectively inspect more than ``max_features`` features. - max_leaf_nodes : int or None, optional (default=None) + max_leaf_nodes : int, default=None Grow trees with ``max_leaf_nodes`` in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. - min_impurity_decrease : float, optional (default=0.) + min_impurity_decrease : float, default=0.0 A node will be split if this split induces a decrease of the impurity greater than or equal to this value. @@ -1833,45 +1904,50 @@ class ExtraTreesRegressor(ForestRegressor): .. versionadded:: 0.19 - min_impurity_split : float, (default=1e-7) + min_impurity_split : float, default=None Threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf. .. deprecated:: 0.19 ``min_impurity_split`` has been deprecated in favor of ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` will change from 1e-7 to 0 in 0.23 and it + ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it will be removed in 0.25. Use ``min_impurity_decrease`` instead. - bootstrap : boolean, optional (default=False) + bootstrap : bool, default=False Whether bootstrap samples are used when building trees. If False, the - whole datset is used to build each tree. + whole dataset is used to build each tree. - oob_score : bool, optional (default=False) + oob_score : bool, default=False Whether to use out-of-bag samples to estimate the R^2 on unseen data. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of jobs to run in parallel. :meth:`fit`, :meth:`predict`, :meth:`decision_path` and :meth:`apply` are all parallelized over the trees. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState, default=None + Controls 3 sources of randomness: + + - the bootstrapping of the samples used when building trees + (if ``bootstrap=True``) + - the sampling of the features to consider when looking for the best + split at each node (if ``max_features < n_features``) + - the draw of the splits for each of the `max_features` - verbose : int, optional (default=0) + See :term:`Glossary ` for details. + + verbose : int, default=0 Controls the verbosity when fitting and predicting. - warm_start : bool, optional (default=False) + warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just fit a whole new forest. See :term:`the Glossary `. - ccp_alpha : non-negative float, optional (default=0.0) + ccp_alpha : non-negative float, default=0.0 Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ``ccp_alpha`` will be chosen. By default, no pruning is performed. See @@ -1900,7 +1976,15 @@ class ExtraTreesRegressor(ForestRegressor): The collection of fitted sub-estimators. feature_importances_ : ndarray of shape (n_features,) - The feature importances (the higher, the more important the feature). + The impurity-based feature importances. + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. n_features_ : int The number of features. @@ -2000,19 +2084,19 @@ class RandomTreesEmbedding(BaseForest): Parameters ---------- - n_estimators : integer, optional (default=10) + n_estimators : int, default=100 Number of trees in the forest. .. versionchanged:: 0.22 The default value of ``n_estimators`` changed from 10 to 100 in 0.22. - max_depth : integer, optional (default=5) + max_depth : int, default=5 The maximum depth of each tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples. - min_samples_split : int, float, optional (default=2) + min_samples_split : int or float, default=2 The minimum number of samples required to split an internal node: - If int, then consider `min_samples_split` as the minimum number. @@ -2023,7 +2107,7 @@ class RandomTreesEmbedding(BaseForest): .. versionchanged:: 0.18 Added float values for fractions. - min_samples_leaf : int, float, optional (default=1) + min_samples_leaf : int or float, default=1 The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least ``min_samples_leaf`` training samples in each of the left and @@ -2038,17 +2122,17 @@ class RandomTreesEmbedding(BaseForest): .. versionchanged:: 0.18 Added float values for fractions. - min_weight_fraction_leaf : float, optional (default=0.) + min_weight_fraction_leaf : float, default=0.0 The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. - max_leaf_nodes : int or None, optional (default=None) + max_leaf_nodes : int, default=None Grow trees with ``max_leaf_nodes`` in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. - min_impurity_decrease : float, optional (default=0.) + min_impurity_decrease : float, default=0.0 A node will be split if this split induces a decrease of the impurity greater than or equal to this value. @@ -2066,65 +2150,59 @@ class RandomTreesEmbedding(BaseForest): .. versionadded:: 0.19 - min_impurity_split : float, (default=1e-7) + min_impurity_split : float, default=None Threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf. .. deprecated:: 0.19 ``min_impurity_split`` has been deprecated in favor of ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` will change from 1e-7 to 0 in 0.23 and it + ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it will be removed in 0.25. Use ``min_impurity_decrease`` instead. - sparse_output : bool, optional (default=True) + sparse_output : bool, default=True Whether or not to return a sparse CSR matrix, as default behavior, or to return a dense array compatible with dense pipeline operators. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of jobs to run in parallel. :meth:`fit`, :meth:`transform`, :meth:`decision_path` and :meth:`apply` are all parallelized over the trees. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState, default=None + Controls the generation of the random `y` used to fit the trees + and the draw of the splits for each feature at the trees' nodes. + See :term:`Glossary ` for details. - verbose : int, optional (default=0) + verbose : int, default=0 Controls the verbosity when fitting and predicting. - warm_start : bool, optional (default=False) + warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just fit a whole new forest. See :term:`the Glossary `. - ccp_alpha : non-negative float, optional (default=0.0) - Complexity parameter used for Minimal Cost-Complexity Pruning. The - subtree with the largest cost complexity that is smaller than - ``ccp_alpha`` will be chosen. By default, no pruning is performed. See - :ref:`minimal_cost_complexity_pruning` for details. - - .. versionadded:: 0.22 - - max_samples : int or float, default=None - If bootstrap is True, the number of samples to draw from X - to train each base estimator. - - - If None (default), then draw `X.shape[0]` samples. - - If int, then draw `max_samples` samples. - - If float, then draw `max_samples * X.shape[0]` samples. Thus, - `max_samples` should be in the interval `(0, 1)`. - - .. versionadded:: 0.22 - Attributes ---------- estimators_ : list of DecisionTreeClassifier The collection of fitted sub-estimators. + Examples + -------- + >>> from sklearn.ensemble import RandomTreesEmbedding + >>> X = [[0,0], [1,0], [0,1], [-1,0], [0,-1]] + >>> random_trees = RandomTreesEmbedding( + ... n_estimators=5, random_state=0, max_depth=1).fit(X) + >>> X_sparse_embedding = random_trees.transform(X) + >>> X_sparse_embedding.toarray() + array([[0., 1., 1., 0., 1., 0., 0., 1., 1., 0.], + [0., 1., 1., 0., 1., 0., 0., 1., 1., 0.], + [0., 1., 0., 1., 0., 1., 0., 1., 0., 1.], + [1., 0., 1., 0., 1., 0., 1., 0., 1., 0.], + [0., 1., 1., 0., 1., 0., 0., 1., 1., 0.]]) + References ---------- .. [1] P. Geurts, D. Ernst., and L. Wehenkel, "Extremely randomized trees", @@ -2151,9 +2229,7 @@ def __init__(self, n_jobs=None, random_state=None, verbose=0, - warm_start=False, - ccp_alpha=0.0, - max_samples=None): + warm_start=False): super().__init__( base_estimator=ExtraTreeRegressor(), n_estimators=n_estimators, @@ -2161,14 +2237,14 @@ def __init__(self, "min_samples_leaf", "min_weight_fraction_leaf", "max_features", "max_leaf_nodes", "min_impurity_decrease", "min_impurity_split", - "random_state", "ccp_alpha"), + "random_state"), bootstrap=False, oob_score=False, n_jobs=n_jobs, random_state=random_state, verbose=verbose, warm_start=warm_start, - max_samples=max_samples) + max_samples=None) self.max_depth = max_depth self.min_samples_split = min_samples_split @@ -2178,7 +2254,6 @@ def __init__(self, self.min_impurity_decrease = min_impurity_decrease self.min_impurity_split = min_impurity_split self.sparse_output = sparse_output - self.ccp_alpha = ccp_alpha def _set_oob_score(self, X, y): raise NotImplementedError("OOB score not supported by tree embedding") @@ -2189,11 +2264,14 @@ def fit(self, X, y=None, sample_weight=None): Parameters ---------- - X : array-like or sparse matrix, shape=(n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Use ``dtype=np.float32`` for maximum efficiency. Sparse matrices are also supported, use sparse ``csc_matrix`` for maximum efficiency. + y : Ignored + Not used, present for API consistency by convention. + sample_weight : array-like of shape (n_samples,), default=None Sample weights. If None, then samples are equally weighted. Splits that would create child nodes with net zero or negative weight are @@ -2215,10 +2293,13 @@ def fit_transform(self, X, y=None, sample_weight=None): Parameters ---------- - X : array-like or sparse matrix, shape=(n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Input data used to build forests. Use ``dtype=np.float32`` for maximum efficiency. + y : Ignored + Not used, present for API consistency by convention. + sample_weight : array-like of shape (n_samples,), default=None Sample weights. If None, then samples are equally weighted. Splits that would create child nodes with net zero or negative weight are @@ -2228,7 +2309,7 @@ def fit_transform(self, X, y=None, sample_weight=None): Returns ------- - X_transformed : sparse matrix, shape=(n_samples, n_out) + X_transformed : sparse matrix of shape (n_samples, n_out) Transformed dataset. """ X = check_array(X, accept_sparse=['csc']) @@ -2250,14 +2331,14 @@ def transform(self, X): Parameters ---------- - X : array-like or sparse matrix, shape=(n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Input data to be transformed. Use ``dtype=np.float32`` for maximum efficiency. Sparse matrices are also supported, use sparse ``csr_matrix`` for maximum efficiency. Returns ------- - X_transformed : sparse matrix, shape=(n_samples, n_out) + X_transformed : sparse matrix of shape (n_samples, n_out) Transformed dataset. """ check_is_fitted(self) diff --git a/sklearn/ensemble/_gb.py b/sklearn/ensemble/_gb.py index 6488d5dd0e776..c9f0b69f57968 100644 --- a/sklearn/ensemble/_gb.py +++ b/sklearn/ensemble/_gb.py @@ -56,1038 +56,11 @@ from ..utils import deprecated from ..utils.fixes import logsumexp from ..utils.stats import _weighted_percentile -from ..utils.validation import check_is_fitted +from ..utils.validation import check_is_fitted, _check_sample_weight from ..utils.multiclass import check_classification_targets from ..exceptions import NotFittedError -# FIXME: 0.23 -# All the losses and corresponding init estimators have been moved to the -# _losses module in 0.21. We deprecate them and keep them here for now in case -# someone has imported them. None of these losses can be used as a parameter -# to a GBDT estimator anyway (loss param only accepts strings). - -@deprecated("QuantileEstimator is deprecated in version " - "0.21 and will be removed in version 0.23.") -class QuantileEstimator: - """An estimator predicting the alpha-quantile of the training targets. - - Parameters - ---------- - alpha : float - The quantile - """ - def __init__(self, alpha=0.9): - if not 0 < alpha < 1.0: - raise ValueError("`alpha` must be in (0, 1.0) but was %r" % alpha) - self.alpha = alpha - - def fit(self, X, y, sample_weight=None): - """Fit the estimator. - - Parameters - ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) - Training data - - y : array, shape (n_samples, n_targets) - Target values. Will be cast to X's dtype if necessary - - sample_weight : numpy array of shape (n_samples,) - Individual weights for each sample - """ - if sample_weight is None: - self.quantile = np.percentile(y, self.alpha * 100.0) - else: - self.quantile = _weighted_percentile(y, sample_weight, - self.alpha * 100.0) - - def predict(self, X): - """Predict labels - - Parameters - ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) - Samples. - - Returns - ------- - y : array, shape (n_samples,) - Returns predicted values. - """ - check_is_fitted(self) - - y = np.empty((X.shape[0], 1), dtype=np.float64) - y.fill(self.quantile) - return y - - -@deprecated("MeanEstimator is deprecated in version " - "0.21 and will be removed in version 0.23.") -class MeanEstimator: - """An estimator predicting the mean of the training targets.""" - def fit(self, X, y, sample_weight=None): - """Fit the estimator. - - Parameters - ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) - Training data - - y : array, shape (n_samples, n_targets) - Target values. Will be cast to X's dtype if necessary - - sample_weight : numpy array of shape (n_samples,) - Individual weights for each sample - """ - if sample_weight is None: - self.mean = np.mean(y) - else: - self.mean = np.average(y, weights=sample_weight) - - def predict(self, X): - """Predict labels - - Parameters - ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) - Samples. - - Returns - ------- - y : array, shape (n_samples,) - Returns predicted values. - """ - check_is_fitted(self) - - y = np.empty((X.shape[0], 1), dtype=np.float64) - y.fill(self.mean) - return y - - -@deprecated("LogOddsEstimator is deprecated in version " - "0.21 and will be removed in version 0.23.") -class LogOddsEstimator: - """An estimator predicting the log odds ratio.""" - scale = 1.0 - - def fit(self, X, y, sample_weight=None): - """Fit the estimator. - - Parameters - ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) - Training data - - y : array, shape (n_samples, n_targets) - Target values. Will be cast to X's dtype if necessary - - sample_weight : numpy array of shape (n_samples,) - Individual weights for each sample - """ - # pre-cond: pos, neg are encoded as 1, 0 - if sample_weight is None: - pos = np.sum(y) - neg = y.shape[0] - pos - else: - pos = np.sum(sample_weight * y) - neg = np.sum(sample_weight * (1 - y)) - - if neg == 0 or pos == 0: - raise ValueError('y contains non binary labels.') - self.prior = self.scale * np.log(pos / neg) - - def predict(self, X): - """Predict labels - - Parameters - ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) - Samples. - - Returns - ------- - y : array, shape (n_samples,) - Returns predicted values. - """ - check_is_fitted(self) - - y = np.empty((X.shape[0], 1), dtype=np.float64) - y.fill(self.prior) - return y - - -@deprecated("ScaledLogOddsEstimator is deprecated in version " - "0.21 and will be removed in version 0.23.") -class ScaledLogOddsEstimator(LogOddsEstimator): - """Log odds ratio scaled by 0.5 -- for exponential loss. """ - scale = 0.5 - - -@deprecated("PriorProbablityEstimator is deprecated in version " - "0.21 and will be removed in version 0.23.") -class PriorProbabilityEstimator: - """An estimator predicting the probability of each - class in the training data. - """ - def fit(self, X, y, sample_weight=None): - """Fit the estimator. - - Parameters - ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) - Training data - - y : array, shape (n_samples, n_targets) - Target values. Will be cast to X's dtype if necessary - - sample_weight : array, shape (n_samples,) - Individual weights for each sample - """ - if sample_weight is None: - sample_weight = np.ones_like(y, dtype=np.float64) - class_counts = np.bincount(y, weights=sample_weight) - self.priors = class_counts / class_counts.sum() - - def predict(self, X): - """Predict labels - - Parameters - ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) - Samples. - - Returns - ------- - y : array, shape (n_samples,) - Returns predicted values. - """ - check_is_fitted(self) - - y = np.empty((X.shape[0], self.priors.shape[0]), dtype=np.float64) - y[:] = self.priors - return y - - -@deprecated("Using ZeroEstimator is deprecated in version " - "0.21 and will be removed in version 0.23.") -class ZeroEstimator: - """An estimator that simply predicts zero. - - .. deprecated:: 0.21 - Using ``ZeroEstimator`` or ``init='zero'`` is deprecated in version - 0.21 and will be removed in version 0.23. - - """ - - def fit(self, X, y, sample_weight=None): - """Fit the estimator. - - Parameters - ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) - Training data - - y : numpy, shape (n_samples, n_targets) - Target values. Will be cast to X's dtype if necessary - - sample_weight : array, shape (n_samples,) - Individual weights for each sample - """ - if np.issubdtype(y.dtype, np.signedinteger): - # classification - self.n_classes = np.unique(y).shape[0] - if self.n_classes == 2: - self.n_classes = 1 - else: - # regression - self.n_classes = 1 - - def predict(self, X): - """Predict labels - - Parameters - ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) - Samples. - - Returns - ------- - y : array, shape (n_samples,) - Returns predicted values. - """ - check_is_fitted(self) - - y = np.empty((X.shape[0], self.n_classes), dtype=np.float64) - y.fill(0.0) - return y - - def predict_proba(self, X): - return self.predict(X) - - -@deprecated("All Losses in sklearn.ensemble.gradient_boosting are " - "deprecated in version " - "0.21 and will be removed in version 0.23.") -class LossFunction(metaclass=ABCMeta): - """Abstract base class for various loss functions. - - Parameters - ---------- - n_classes : int - Number of classes - - Attributes - ---------- - K : int - The number of regression trees to be induced; - 1 for regression and binary classification; - ``n_classes`` for multi-class classification. - """ - - is_multi_class = False - - def __init__(self, n_classes): - self.K = n_classes - - def init_estimator(self): - """Default ``init`` estimator for loss function. """ - raise NotImplementedError() - - @abstractmethod - def __call__(self, y, pred, sample_weight=None): - """Compute the loss. - - Parameters - ---------- - y : array, shape (n_samples,) - True labels - - pred : array, shape (n_samples,) - Predicted labels - - sample_weight : array-like, shape (n_samples,), optional - Sample weights. - """ - - @abstractmethod - def negative_gradient(self, y, y_pred, **kargs): - """Compute the negative gradient. - - Parameters - ---------- - y : array, shape (n_samples,) - The target labels. - - y_pred : array, shape (n_samples,) - The predictions. - """ - - def update_terminal_regions(self, tree, X, y, residual, y_pred, - sample_weight, sample_mask, - learning_rate=0.1, k=0): - """Update the terminal regions (=leaves) of the given tree and - updates the current predictions of the model. Traverses tree - and invokes template method `_update_terminal_region`. - - Parameters - ---------- - tree : tree.Tree - The tree object. - X : array, shape (n, m) - The data array. - y : array, shape (n,) - The target labels. - residual : array, shape (n,) - The residuals (usually the negative gradient). - y_pred : array, shape (n,) - The predictions. - sample_weight : array, shape (n,) - The weight of each sample. - sample_mask : array, shape (n,) - The sample mask to be used. - learning_rate : float, default=0.1 - learning rate shrinks the contribution of each tree by - ``learning_rate``. - k : int, default 0 - The index of the estimator being updated. - - """ - # compute leaf for each sample in ``X``. - terminal_regions = tree.apply(X) - - # mask all which are not in sample mask. - masked_terminal_regions = terminal_regions.copy() - masked_terminal_regions[~sample_mask] = -1 - - # update each leaf (= perform line search) - for leaf in np.where(tree.children_left == TREE_LEAF)[0]: - self._update_terminal_region(tree, masked_terminal_regions, - leaf, X, y, residual, - y_pred[:, k], sample_weight) - - # update predictions (both in-bag and out-of-bag) - y_pred[:, k] += (learning_rate - * tree.value[:, 0, 0].take(terminal_regions, axis=0)) - - @abstractmethod - def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, - residual, pred, sample_weight): - """Template method for updating terminal regions (=leaves). """ - - -@deprecated("All Losses in sklearn.ensemble.gradient_boosting are " - "deprecated in version " - "0.21 and will be removed in version 0.23.") -class RegressionLossFunction(LossFunction, metaclass=ABCMeta): - """Base class for regression loss functions. - - Parameters - ---------- - n_classes : int - Number of classes - """ - def __init__(self, n_classes): - if n_classes != 1: - raise ValueError("``n_classes`` must be 1 for regression but " - "was %r" % n_classes) - super().__init__(n_classes) - - -@deprecated("All Losses in sklearn.ensemble.gradient_boosting are " - "deprecated in version " - "0.21 and will be removed in version 0.23.") -class LeastSquaresError(RegressionLossFunction): - """Loss function for least squares (LS) estimation. - Terminal regions need not to be updated for least squares. - - Parameters - ---------- - n_classes : int - Number of classes - """ - - def init_estimator(self): - return MeanEstimator() - - def __call__(self, y, pred, sample_weight=None): - """Compute the least squares loss. - - Parameters - ---------- - y : array, shape (n_samples,) - True labels - - pred : array, shape (n_samples,) - Predicted labels - - sample_weight : array-like, shape (n_samples,), optional - Sample weights. - """ - if sample_weight is None: - return np.mean((y - pred.ravel()) ** 2.0) - else: - return (1.0 / sample_weight.sum() * - np.sum(sample_weight * ((y - pred.ravel()) ** 2.0))) - - def negative_gradient(self, y, pred, **kargs): - """Compute the negative gradient. - - Parameters - ---------- - y : array, shape (n_samples,) - The target labels. - - pred : array, shape (n_samples,) - The predictions. - """ - return y - pred.ravel() - - def update_terminal_regions(self, tree, X, y, residual, y_pred, - sample_weight, sample_mask, - learning_rate=0.1, k=0): - """Least squares does not need to update terminal regions. - - But it has to update the predictions. - - Parameters - ---------- - tree : tree.Tree - The tree object. - X : array, shape (n, m) - The data array. - y : array, shape (n,) - The target labels. - residual : array, shape (n,) - The residuals (usually the negative gradient). - y_pred : array, shape (n,) - The predictions. - sample_weight : array, shape (n,) - The weight of each sample. - sample_mask : array, shape (n,) - The sample mask to be used. - learning_rate : float, default=0.1 - learning rate shrinks the contribution of each tree by - ``learning_rate``. - k : int, default 0 - The index of the estimator being updated. - """ - # update predictions - y_pred[:, k] += learning_rate * tree.predict(X).ravel() - - def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, - residual, pred, sample_weight): - pass - - -@deprecated("All Losses in sklearn.ensemble.gradient_boosting are " - "deprecated in version " - "0.21 and will be removed in version 0.23.") -class LeastAbsoluteError(RegressionLossFunction): - """Loss function for least absolute deviation (LAD) regression. - - Parameters - ---------- - n_classes : int - Number of classes - """ - def init_estimator(self): - return QuantileEstimator(alpha=0.5) - - def __call__(self, y, pred, sample_weight=None): - """Compute the least absolute error. - - Parameters - ---------- - y : array, shape (n_samples,) - True labels - - pred : array, shape (n_samples,) - Predicted labels - - sample_weight : array-like, shape (n_samples,), optional - Sample weights. - """ - if sample_weight is None: - return np.abs(y - pred.ravel()).mean() - else: - return (1.0 / sample_weight.sum() * - np.sum(sample_weight * np.abs(y - pred.ravel()))) - - def negative_gradient(self, y, pred, **kargs): - """Compute the negative gradient. - - 1.0 if y - pred > 0.0 else -1.0 - - Parameters - ---------- - y : array, shape (n_samples,) - The target labels. - - pred : array, shape (n_samples,) - The predictions. - """ - pred = pred.ravel() - return 2.0 * (y - pred > 0.0) - 1.0 - - def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, - residual, pred, sample_weight): - """LAD updates terminal regions to median estimates. """ - terminal_region = np.where(terminal_regions == leaf)[0] - sample_weight = sample_weight.take(terminal_region, axis=0) - diff = y.take(terminal_region, axis=0) - pred.take(terminal_region, axis=0) - tree.value[leaf, 0, 0] = _weighted_percentile(diff, sample_weight, percentile=50) - - -@deprecated("All Losses in sklearn.ensemble.gradient_boosting are " - "deprecated in version " - "0.21 and will be removed in version 0.23.") -class HuberLossFunction(RegressionLossFunction): - """Huber loss function for robust regression. - - M-Regression proposed in Friedman 2001. - - References - ---------- - J. Friedman, Greedy Function Approximation: A Gradient Boosting - Machine, The Annals of Statistics, Vol. 29, No. 5, 2001. - - Parameters - ---------- - n_classes : int - Number of classes - - alpha : float - Percentile at which to extract score - """ - - def __init__(self, n_classes, alpha=0.9): - super().__init__(n_classes) - self.alpha = alpha - self.gamma = None - - def init_estimator(self): - return QuantileEstimator(alpha=0.5) - - def __call__(self, y, pred, sample_weight=None): - """Compute the Huber loss. - - Parameters - ---------- - y : array, shape (n_samples,) - True labels - - pred : array, shape (n_samples,) - Predicted labels - - sample_weight : array-like, shape (n_samples,), optional - Sample weights. - """ - pred = pred.ravel() - diff = y - pred - gamma = self.gamma - if gamma is None: - if sample_weight is None: - gamma = np.percentile(np.abs(diff), self.alpha * 100) - else: - gamma = _weighted_percentile(np.abs(diff), sample_weight, self.alpha * 100) - - gamma_mask = np.abs(diff) <= gamma - if sample_weight is None: - sq_loss = np.sum(0.5 * diff[gamma_mask] ** 2.0) - lin_loss = np.sum(gamma * (np.abs(diff[~gamma_mask]) - gamma / 2.0)) - loss = (sq_loss + lin_loss) / y.shape[0] - else: - sq_loss = np.sum(0.5 * sample_weight[gamma_mask] * diff[gamma_mask] ** 2.0) - lin_loss = np.sum(gamma * sample_weight[~gamma_mask] * - (np.abs(diff[~gamma_mask]) - gamma / 2.0)) - loss = (sq_loss + lin_loss) / sample_weight.sum() - return loss - - def negative_gradient(self, y, pred, sample_weight=None, **kargs): - """Compute the negative gradient. - - Parameters - ---------- - y : array, shape (n_samples,) - The target labels. - - pred : array, shape (n_samples,) - The predictions. - - sample_weight : array-like, shape (n_samples,), optional - Sample weights. - """ - pred = pred.ravel() - diff = y - pred - if sample_weight is None: - gamma = np.percentile(np.abs(diff), self.alpha * 100) - else: - gamma = _weighted_percentile(np.abs(diff), sample_weight, self.alpha * 100) - gamma_mask = np.abs(diff) <= gamma - residual = np.zeros((y.shape[0],), dtype=np.float64) - residual[gamma_mask] = diff[gamma_mask] - residual[~gamma_mask] = gamma * np.sign(diff[~gamma_mask]) - self.gamma = gamma - return residual - - def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, - residual, pred, sample_weight): - terminal_region = np.where(terminal_regions == leaf)[0] - sample_weight = sample_weight.take(terminal_region, axis=0) - gamma = self.gamma - diff = (y.take(terminal_region, axis=0) - - pred.take(terminal_region, axis=0)) - median = _weighted_percentile(diff, sample_weight, percentile=50) - diff_minus_median = diff - median - tree.value[leaf, 0] = median + np.mean( - np.sign(diff_minus_median) * - np.minimum(np.abs(diff_minus_median), gamma)) - - -@deprecated("All Losses in sklearn.ensemble.gradient_boosting are " - "deprecated in version " - "0.21 and will be removed in version 0.23.") -class QuantileLossFunction(RegressionLossFunction): - """Loss function for quantile regression. - - Quantile regression allows to estimate the percentiles - of the conditional distribution of the target. - - Parameters - ---------- - n_classes : int - Number of classes. - - alpha : float, optional (default = 0.9) - The percentile - """ - def __init__(self, n_classes, alpha=0.9): - super().__init__(n_classes) - self.alpha = alpha - self.percentile = alpha * 100.0 - - def init_estimator(self): - return QuantileEstimator(self.alpha) - - def __call__(self, y, pred, sample_weight=None): - """Compute the Quantile loss. - - Parameters - ---------- - y : array, shape (n_samples,) - True labels - - pred : array, shape (n_samples,) - Predicted labels - - sample_weight : array-like, shape (n_samples,), optional - Sample weights. - """ - pred = pred.ravel() - diff = y - pred - alpha = self.alpha - - mask = y > pred - if sample_weight is None: - loss = (alpha * diff[mask].sum() - - (1.0 - alpha) * diff[~mask].sum()) / y.shape[0] - else: - loss = ((alpha * np.sum(sample_weight[mask] * diff[mask]) - - (1.0 - alpha) * np.sum(sample_weight[~mask] * diff[~mask])) / - sample_weight.sum()) - return loss - - def negative_gradient(self, y, pred, **kargs): - """Compute the negative gradient. - - Parameters - ---------- - y : array, shape (n_samples,) - The target labels. - - pred : array, shape (n_samples,) - The predictions. - """ - alpha = self.alpha - pred = pred.ravel() - mask = y > pred - return (alpha * mask) - ((1.0 - alpha) * ~mask) - - def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, - residual, pred, sample_weight): - terminal_region = np.where(terminal_regions == leaf)[0] - diff = (y.take(terminal_region, axis=0) - - pred.take(terminal_region, axis=0)) - sample_weight = sample_weight.take(terminal_region, axis=0) - - val = _weighted_percentile(diff, sample_weight, self.percentile) - tree.value[leaf, 0] = val - - -@deprecated("All Losses in sklearn.ensemble.gradient_boosting are " - "deprecated in version " - "0.21 and will be removed in version 0.23.") -class ClassificationLossFunction(LossFunction, metaclass=ABCMeta): - """Base class for classification loss functions. """ - - def _score_to_proba(self, score): - """Template method to convert scores to probabilities. - - the does not support probabilities raises AttributeError. - """ - raise TypeError('%s does not support predict_proba' % type(self).__name__) - - @abstractmethod - def _score_to_decision(self, score): - """Template method to convert scores to decisions. - - Returns int arrays. - """ - - -@deprecated("All Losses in sklearn.ensemble.gradient_boosting are " - "deprecated in version " - "0.21 and will be removed in version 0.23.") -class BinomialDeviance(ClassificationLossFunction): - """Binomial deviance loss function for binary classification. - - Binary classification is a special case; here, we only need to - fit one tree instead of ``n_classes`` trees. - - Parameters - ---------- - n_classes : int - Number of classes. - """ - def __init__(self, n_classes): - if n_classes != 2: - raise ValueError("{0:s} requires 2 classes; got {1:d} class(es)" - .format(self.__class__.__name__, n_classes)) - # we only need to fit one tree for binary clf. - super().__init__(1) - - def init_estimator(self): - return LogOddsEstimator() - - def __call__(self, y, pred, sample_weight=None): - """Compute the deviance (= 2 * negative log-likelihood). - - Parameters - ---------- - y : array, shape (n_samples,) - True labels - - pred : array, shape (n_samples,) - Predicted labels - - sample_weight : array-like, shape (n_samples,), optional - Sample weights. - """ - # logaddexp(0, v) == log(1.0 + exp(v)) - pred = pred.ravel() - if sample_weight is None: - return -2.0 * np.mean((y * pred) - np.logaddexp(0.0, pred)) - else: - return (-2.0 / sample_weight.sum() * - np.sum(sample_weight * ((y * pred) - np.logaddexp(0.0, pred)))) - - def negative_gradient(self, y, pred, **kargs): - """Compute the residual (= negative gradient). - - Parameters - ---------- - y : array, shape (n_samples,) - True labels - - pred : array, shape (n_samples,) - Predicted labels - """ - return y - expit(pred.ravel()) - - def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, - residual, pred, sample_weight): - """Make a single Newton-Raphson step. - - our node estimate is given by: - - sum(w * (y - prob)) / sum(w * prob * (1 - prob)) - - we take advantage that: y - prob = residual - """ - terminal_region = np.where(terminal_regions == leaf)[0] - residual = residual.take(terminal_region, axis=0) - y = y.take(terminal_region, axis=0) - sample_weight = sample_weight.take(terminal_region, axis=0) - - numerator = np.sum(sample_weight * residual) - denominator = np.sum(sample_weight * (y - residual) * (1 - y + residual)) - - # prevents overflow and division by zero - if abs(denominator) < 1e-150: - tree.value[leaf, 0, 0] = 0.0 - else: - tree.value[leaf, 0, 0] = numerator / denominator - - def _score_to_proba(self, score): - proba = np.ones((score.shape[0], 2), dtype=np.float64) - proba[:, 1] = expit(score.ravel()) - proba[:, 0] -= proba[:, 1] - return proba - - def _score_to_decision(self, score): - proba = self._score_to_proba(score) - return np.argmax(proba, axis=1) - - -@deprecated("All Losses in sklearn.ensemble.gradient_boosting are " - "deprecated in version " - "0.21 and will be removed in version 0.23.") -class MultinomialDeviance(ClassificationLossFunction): - """Multinomial deviance loss function for multi-class classification. - - For multi-class classification we need to fit ``n_classes`` trees at - each stage. - - Parameters - ---------- - n_classes : int - Number of classes - """ - - is_multi_class = True - - def __init__(self, n_classes): - if n_classes < 3: - raise ValueError("{0:s} requires more than 2 classes.".format( - self.__class__.__name__)) - super().__init__(n_classes) - - def init_estimator(self): - return PriorProbabilityEstimator() - - def __call__(self, y, pred, sample_weight=None): - """Compute the Multinomial deviance. - - Parameters - ---------- - y : array, shape (n_samples,) - True labels - - pred : array, shape (n_samples,) - Predicted labels - - sample_weight : array-like, shape (n_samples,), optional - Sample weights. - """ - # create one-hot label encoding - Y = np.zeros((y.shape[0], self.K), dtype=np.float64) - for k in range(self.K): - Y[:, k] = y == k - - if sample_weight is None: - return np.sum(-1 * (Y * pred).sum(axis=1) + - logsumexp(pred, axis=1)) - else: - return np.sum(-1 * sample_weight * (Y * pred).sum(axis=1) + - logsumexp(pred, axis=1)) - - def negative_gradient(self, y, pred, k=0, **kwargs): - """Compute negative gradient for the ``k``-th class. - - Parameters - ---------- - y : array, shape (n_samples,) - The target labels. - - pred : array, shape (n_samples,) - The predictions. - - k : int, optional (default=0) - The index of the class - """ - return y - np.nan_to_num(np.exp(pred[:, k] - - logsumexp(pred, axis=1))) - - def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, - residual, pred, sample_weight): - """Make a single Newton-Raphson step. """ - terminal_region = np.where(terminal_regions == leaf)[0] - residual = residual.take(terminal_region, axis=0) - y = y.take(terminal_region, axis=0) - sample_weight = sample_weight.take(terminal_region, axis=0) - - numerator = np.sum(sample_weight * residual) - numerator *= (self.K - 1) / self.K - - denominator = np.sum(sample_weight * (y - residual) * - (1.0 - y + residual)) - - # prevents overflow and division by zero - if abs(denominator) < 1e-150: - tree.value[leaf, 0, 0] = 0.0 - else: - tree.value[leaf, 0, 0] = numerator / denominator - - def _score_to_proba(self, score): - return np.nan_to_num( - np.exp(score - (logsumexp(score, axis=1)[:, np.newaxis]))) - - def _score_to_decision(self, score): - proba = self._score_to_proba(score) - return np.argmax(proba, axis=1) - - -@deprecated("All Losses in sklearn.ensemble.gradient_boosting are " - "deprecated in version " - "0.21 and will be removed in version 0.23.") -class ExponentialLoss(ClassificationLossFunction): - """Exponential loss function for binary classification. - - Same loss as AdaBoost. - - References - ---------- - Greg Ridgeway, Generalized Boosted Models: A guide to the gbm package, 2007 - - Parameters - ---------- - n_classes : int - Number of classes. - """ - def __init__(self, n_classes): - if n_classes != 2: - raise ValueError("{0:s} requires 2 classes; got {1:d} class(es)" - .format(self.__class__.__name__, n_classes)) - # we only need to fit one tree for binary clf. - super().__init__(1) - - def init_estimator(self): - return ScaledLogOddsEstimator() - - def __call__(self, y, pred, sample_weight=None): - """Compute the exponential loss - - Parameters - ---------- - y : array, shape (n_samples,) - True labels - - pred : array, shape (n_samples,) - Predicted labels - - sample_weight : array-like, shape (n_samples,), optional - Sample weights. - """ - pred = pred.ravel() - if sample_weight is None: - return np.mean(np.exp(-(2. * y - 1.) * pred)) - else: - return (1.0 / sample_weight.sum() * - np.sum(sample_weight * np.exp(-(2 * y - 1) * pred))) - - def negative_gradient(self, y, pred, **kargs): - """Compute the residual (= negative gradient). - - Parameters - ---------- - y : array, shape (n_samples,) - True labels - - pred : array, shape (n_samples,) - Predicted labels - """ - y_ = -(2. * y - 1.) - return y_ * np.exp(y_ * pred.ravel()) - - def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, - residual, pred, sample_weight): - terminal_region = np.where(terminal_regions == leaf)[0] - pred = pred.take(terminal_region, axis=0) - y = y.take(terminal_region, axis=0) - sample_weight = sample_weight.take(terminal_region, axis=0) - - y_ = 2. * y - 1. - - numerator = np.sum(y_ * sample_weight * np.exp(-y_ * pred)) - denominator = np.sum(sample_weight * np.exp(-y_ * pred)) - - # prevents overflow and division by zero - if abs(denominator) < 1e-150: - tree.value[leaf, 0, 0] = 0.0 - else: - tree.value[leaf, 0, 0] = numerator / denominator - - def _score_to_proba(self, score): - proba = np.ones((score.shape[0], 2), dtype=np.float64) - proba[:, 1] = expit(2.0 * score.ravel()) - proba[:, 0] -= proba[:, 1] - return proba - - def _score_to_decision(self, score): - return (score.ravel() >= 0.0).astype(np.int) - - class VerboseReporter: """Reports verbose output to stdout. @@ -1110,7 +83,7 @@ def init(self, est, begin_at_stage=0): est : Estimator The estimator - begin_at_stage : int + begin_at_stage : int, default=0 stage at which to begin reporting """ # header fields and line format str @@ -1401,24 +374,24 @@ def fit(self, X, y, sample_weight=None, monitor=None): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) Target values (strings or integers in classification, real numbers in regression) For classification, labels must correspond to classes. - sample_weight : array-like, shape (n_samples,) or None + sample_weight : array-like of shape (n_samples,), default=None Sample weights. If None, then samples are equally weighted. Splits that would create child nodes with net zero or negative weight are ignored while searching for a split in each node. In the case of classification, splits are also ignored if they would result in any single class carrying a negative weight in either child node. - monitor : callable, optional + monitor : callable, default=None The monitor is called after each iteration with the current iteration, a reference to the estimator and the local variables of ``_fit_stages`` as keyword arguments ``callable(i, self, @@ -1438,17 +411,13 @@ def fit(self, X, y, sample_weight=None, monitor=None): # Check input # Since check_array converts both X and y to the same dtype, but the # trees use different types for X and y, checking them separately. - X = check_array(X, accept_sparse=['csr', 'csc', 'coo'], dtype=DTYPE) + X = self._validate_data(X, accept_sparse=['csr', 'csc', 'coo'], + dtype=DTYPE) n_samples, self.n_features_ = X.shape sample_weight_is_none = sample_weight is None - if sample_weight_is_none: - sample_weight = np.ones(n_samples, dtype=np.float32) - else: - sample_weight = column_or_1d(sample_weight, warn=True) - sample_weight_is_none = False - check_consistent_length(X, y, sample_weight) + sample_weight = _check_sample_weight(sample_weight, X) y = check_array(y, accept_sparse='csc', ensure_2d=False, dtype=None) y = column_or_1d(y, warn=True) @@ -1636,7 +605,7 @@ def _make_estimator(self, append=True): raise NotImplementedError() def _raw_predict_init(self, X): - """Check input and compute raw predictions of the init estimtor.""" + """Check input and compute raw predictions of the init estimator.""" self._check_initialized() X = self.estimators_[0, 0]._validate_X_predict(X, check_input=True) if X.shape[1] != self.n_features_: @@ -1665,14 +634,14 @@ def _staged_raw_predict(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. Returns ------- - raw_predictions : generator of array, shape (n_samples, k) + raw_predictions : generator of ndarray of shape (n_samples, k) The raw predictions of the input samples. The order of the classes corresponds to that in the attribute :term:`classes_`. Regression and binary classification are special cases with @@ -1687,8 +656,16 @@ def _staged_raw_predict(self, X): @property def feature_importances_(self): - """Return the feature importances (the higher, the more important the - feature). + """The impurity-based feature importances. + + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. Returns ------- @@ -1719,21 +696,19 @@ def _compute_partial_dependence_recursion(self, grid, target_features): Parameters ---------- - grid : ndarray, shape (n_samples, n_target_features) + grid : ndarray of shape (n_samples, n_target_features) The grid points on which the partial dependence should be evaluated. - target_features : ndarray, shape (n_target_features) + target_features : ndarray of shape (n_target_features,) The set of target features for which the partial dependence should be evaluated. Returns ------- - averaged_predictions : ndarray, shape \ + averaged_predictions : ndarray of shape \ (n_trees_per_iteration, n_samples) The value of the partial dependence function on each grid point. """ - check_is_fitted(self, - msg="'estimator' parameter must be a fitted estimator") if self.init is not None: warnings.warn( 'Using recursion method with a non-constant init predictor ' @@ -1770,14 +745,14 @@ def apply(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, its dtype will be converted to ``dtype=np.float32``. If a sparse matrix is provided, it will be converted to a sparse ``csr_matrix``. Returns ------- - X_leaves : array-like, shape (n_samples, n_estimators, n_classes) + X_leaves : array-like of shape (n_samples, n_estimators, n_classes) For each datapoint x in X and for each tree in the ensemble, return the index of the leaf x ends up in each estimator. In the case of binary classification n_classes is 1. @@ -1813,39 +788,39 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): Parameters ---------- - loss : {'deviance', 'exponential'}, optional (default='deviance') + loss : {'deviance', 'exponential'}, default='deviance' loss function to be optimized. 'deviance' refers to deviance (= logistic regression) for classification with probabilistic outputs. For loss 'exponential' gradient boosting recovers the AdaBoost algorithm. - learning_rate : float, optional (default=0.1) + learning_rate : float, default=0.1 learning rate shrinks the contribution of each tree by `learning_rate`. There is a trade-off between learning_rate and n_estimators. - n_estimators : int (default=100) + n_estimators : int, default=100 The number of boosting stages to perform. Gradient boosting is fairly robust to over-fitting so a large number usually results in better performance. - subsample : float, optional (default=1.0) + subsample : float, default=1.0 The fraction of samples to be used for fitting the individual base learners. If smaller than 1.0 this results in Stochastic Gradient Boosting. `subsample` interacts with the parameter `n_estimators`. Choosing `subsample < 1.0` leads to a reduction of variance and an increase in bias. - criterion : string, optional (default="friedman_mse") + criterion : {'friedman_mse', 'mse', 'mae'}, default='friedman_mse' The function to measure the quality of a split. Supported criteria - are "friedman_mse" for the mean squared error with improvement - score by Friedman, "mse" for mean squared error, and "mae" for - the mean absolute error. The default value of "friedman_mse" is + are 'friedman_mse' for the mean squared error with improvement + score by Friedman, 'mse' for mean squared error, and 'mae' for + the mean absolute error. The default value of 'friedman_mse' is generally the best as it can provide a better approximation in some cases. .. versionadded:: 0.18 - min_samples_split : int, float, optional (default=2) + min_samples_split : int or float, default=2 The minimum number of samples required to split an internal node: - If int, then consider `min_samples_split` as the minimum number. @@ -1856,7 +831,7 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): .. versionchanged:: 0.18 Added float values for fractions. - min_samples_leaf : int, float, optional (default=1) + min_samples_leaf : int or float, default=1 The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least ``min_samples_leaf`` training samples in each of the left and @@ -1871,18 +846,18 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): .. versionchanged:: 0.18 Added float values for fractions. - min_weight_fraction_leaf : float, optional (default=0.) + min_weight_fraction_leaf : float, default=0.0 The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. - max_depth : integer, optional (default=3) + max_depth : int, default=3 maximum depth of the individual regression estimators. The maximum depth limits the number of nodes in the tree. Tune this parameter for best performance; the best value depends on the interaction of the input variables. - min_impurity_decrease : float, optional (default=0.) + min_impurity_decrease : float, default=0.0 A node will be split if this split induces a decrease of the impurity greater than or equal to this value. @@ -1900,38 +875,42 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): .. versionadded:: 0.19 - min_impurity_split : float, (default=1e-7) + min_impurity_split : float, default=None Threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf. .. deprecated:: 0.19 ``min_impurity_split`` has been deprecated in favor of ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` will change from 1e-7 to 0 in 0.23 and it + ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it will be removed in 0.25. Use ``min_impurity_decrease`` instead. - init : estimator or 'zero', optional (default=None) + init : estimator or 'zero', default=None An estimator object that is used to compute the initial predictions. ``init`` has to provide :meth:`fit` and :meth:`predict_proba`. If 'zero', the initial raw predictions are set to zero. By default, a ``DummyEstimator`` predicting the classes priors is used. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - max_features : int, float, string or None, optional (default=None) + random_state : int or RandomState, default=None + Controls the random seed given to each Tree estimator at each + boosting iteration. + In addition, it controls the random permutation of the features at + each split (see Notes for more details). + It also controls the random spliting of the training data to obtain a + validation set if `n_iter_no_change` is not None. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. + + max_features : {'auto', 'sqrt', 'log2'}, int or float, default=None The number of features to consider when looking for the best split: - If int, then consider `max_features` features at each split. - If float, then `max_features` is a fraction and `int(max_features * n_features)` features are considered at each split. - - If "auto", then `max_features=sqrt(n_features)`. - - If "sqrt", then `max_features=sqrt(n_features)`. - - If "log2", then `max_features=log2(n_features)`. + - If 'auto', then `max_features=sqrt(n_features)`. + - If 'sqrt', then `max_features=sqrt(n_features)`. + - If 'log2', then `max_features=log2(n_features)`. - If None, then `max_features=n_features`. Choosing `max_features < n_features` leads to a reduction of variance @@ -1941,17 +920,17 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): valid partition of the node samples is found, even if it requires to effectively inspect more than ``max_features`` features. - verbose : int, default: 0 + verbose : int, default=0 Enable verbose output. If 1 then it prints progress and performance once in a while (the more trees the lower the frequency). If greater than 1 then it prints progress and performance for every tree. - max_leaf_nodes : int or None, optional (default=None) + max_leaf_nodes : int, default=None Grow trees with ``max_leaf_nodes`` in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. - warm_start : bool, default: False + warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just erase the previous solution. See :term:`the Glossary `. @@ -1961,14 +940,14 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): .. deprecated :: 0.22 - validation_fraction : float, optional, default 0.1 + validation_fraction : float, default=0.1 The proportion of training data to set aside as validation set for early stopping. Must be between 0 and 1. Only used if ``n_iter_no_change`` is set to an integer. .. versionadded:: 0.20 - n_iter_no_change : int, default None + n_iter_no_change : int, default=None ``n_iter_no_change`` is used to decide if early stopping will be used to terminate training when validation score is not improving. By default it is set to None to disable early stopping. If set to a @@ -1979,14 +958,14 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): .. versionadded:: 0.20 - tol : float, optional, default 1e-4 + tol : float, default=1e-4 Tolerance for the early stopping. When the loss is not improving by at least tol for ``n_iter_no_change`` iterations (if set to a number), the training stops. .. versionadded:: 0.20 - ccp_alpha : non-negative float, optional (default=0.0) + ccp_alpha : non-negative float, default=0.0 Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ``ccp_alpha`` will be chosen. By default, no pruning is performed. See @@ -2003,17 +982,25 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): .. versionadded:: 0.20 - feature_importances_ : array, shape (n_features,) - The feature importances (the higher, the more important the feature). + feature_importances_ : ndarray of shape (n_features,) + The impurity-based feature importances. + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. - oob_improvement_ : array, shape (n_estimators,) + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + + oob_improvement_ : ndarray of shape (n_estimators,) The improvement in loss (= deviance) on the out-of-bag samples relative to the previous iteration. ``oob_improvement_[0]`` is the improvement in loss of the first stage over the ``init`` estimator. Only available if ``subsample < 1.0`` - train_score_ : array, shape (n_estimators,) + train_score_ : ndarray of shape (n_estimators,) The i-th score ``train_score_[i]`` is the deviance (= loss) of the model at iteration ``i`` on the in-bag sample. If ``subsample == 1`` this is the deviance on the training data. @@ -2025,14 +1012,23 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): The estimator that provides the initial predictions. Set via the ``init`` argument or ``loss.init_estimator``. - estimators_ : ndarray of DecisionTreeRegressor,\ + estimators_ : ndarray of DecisionTreeRegressor of \ shape (n_estimators, ``loss_.K``) The collection of fitted sub-estimators. ``loss_.K`` is 1 for binary classification, otherwise n_classes. - classes_ : array of shape (n_classes,) + classes_ : ndarray of shape (n_classes,) The classes labels. + n_features_ : int + The number of data features. + + n_classes_ : int + The number of classes. + + max_features_ : int + The inferred value of max_features. + Notes ----- The features are always randomly permuted at each split. Therefore, @@ -2042,6 +1038,22 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): split. To obtain a deterministic behaviour during fitting, ``random_state`` has to be fixed. + Examples + -------- + >>> from sklearn.datasets import make_classification + >>> from sklearn.ensemble import GradientBoostingClassifier + >>> from sklearn.model_selection import train_test_split + >>> X, y = make_classification(random_state=0) + >>> X_train, X_test, y_train, y_test = train_test_split( + ... X, y, random_state=0) + >>> clf = GradientBoostingClassifier(random_state=0) + >>> clf.fit(X_train, y_train) + GradientBoostingClassifier(random_state=0) + >>> clf.predict(X_test[:2]) + array([1, 0]) + >>> clf.score(X_test, y_test) + 0.88 + See also -------- sklearn.ensemble.HistGradientBoostingClassifier, @@ -2103,14 +1115,14 @@ def decision_function(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. Returns ------- - score : array, shape (n_samples, n_classes) or (n_samples,) + score : ndarray of shape (n_samples, n_classes) or (n_samples,) The decision function of the input samples, which corresponds to the raw values predicted from the trees of the ensemble . The order of the classes corresponds to that in the attribute @@ -2131,14 +1143,14 @@ def staged_decision_function(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. Returns ------- - score : generator of array, shape (n_samples, k) + score : generator of ndarray of shape (n_samples, k) The decision function of the input samples, which corresponds to the raw values predicted from the trees of the ensemble . The classes corresponds to that in the attribute :term:`classes_`. @@ -2152,14 +1164,14 @@ def predict(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. Returns ------- - y : array, shape (n_samples,) + y : ndarray of shape (n_samples,) The predicted values. """ raw_predictions = self.decision_function(X) @@ -2175,14 +1187,14 @@ def staged_predict(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. Returns ------- - y : generator of array of shape (n_samples,) + y : generator of ndarray of shape (n_samples,) The predicted value of the input samples. """ for raw_predictions in self._staged_raw_predict(X): @@ -2195,7 +1207,7 @@ def predict_proba(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. @@ -2207,7 +1219,7 @@ def predict_proba(self, X): Returns ------- - p : array, shape (n_samples, n_classes) + p : ndarray of shape (n_samples, n_classes) The class probabilities of the input samples. The order of the classes corresponds to that in the attribute :term:`classes_`. """ @@ -2225,7 +1237,7 @@ def predict_log_proba(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. @@ -2237,7 +1249,7 @@ def predict_log_proba(self, X): Returns ------- - p : array, shape (n_samples, n_classes) + p : ndarray of shape (n_samples, n_classes) The class log-probabilities of the input samples. The order of the classes corresponds to that in the attribute :term:`classes_`. """ @@ -2252,14 +1264,14 @@ def staged_predict_proba(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. Returns ------- - y : generator of array of shape (n_samples,) + y : generator of ndarray of shape (n_samples,) The predicted value of the input samples. """ try: @@ -2284,30 +1296,30 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): Parameters ---------- - loss : {'ls', 'lad', 'huber', 'quantile'}, optional (default='ls') + loss : {'ls', 'lad', 'huber', 'quantile'}, default='ls' loss function to be optimized. 'ls' refers to least squares regression. 'lad' (least absolute deviation) is a highly robust loss function solely based on order information of the input variables. 'huber' is a combination of the two. 'quantile' allows quantile regression (use `alpha` to specify the quantile). - learning_rate : float, optional (default=0.1) + learning_rate : float, default=0.1 learning rate shrinks the contribution of each tree by `learning_rate`. There is a trade-off between learning_rate and n_estimators. - n_estimators : int (default=100) + n_estimators : int, default=100 The number of boosting stages to perform. Gradient boosting is fairly robust to over-fitting so a large number usually results in better performance. - subsample : float, optional (default=1.0) + subsample : float, default=1.0 The fraction of samples to be used for fitting the individual base learners. If smaller than 1.0 this results in Stochastic Gradient Boosting. `subsample` interacts with the parameter `n_estimators`. Choosing `subsample < 1.0` leads to a reduction of variance and an increase in bias. - criterion : string, optional (default="friedman_mse") + criterion : {'friedman_mse', 'mse', 'mae'}, default='friedman_mse' The function to measure the quality of a split. Supported criteria are "friedman_mse" for the mean squared error with improvement score by Friedman, "mse" for mean squared error, and "mae" for @@ -2317,7 +1329,7 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): .. versionadded:: 0.18 - min_samples_split : int, float, optional (default=2) + min_samples_split : int or float, default=2 The minimum number of samples required to split an internal node: - If int, then consider `min_samples_split` as the minimum number. @@ -2328,7 +1340,7 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): .. versionchanged:: 0.18 Added float values for fractions. - min_samples_leaf : int, float, optional (default=1) + min_samples_leaf : int or float, default=1 The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least ``min_samples_leaf`` training samples in each of the left and @@ -2343,18 +1355,18 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): .. versionchanged:: 0.18 Added float values for fractions. - min_weight_fraction_leaf : float, optional (default=0.) + min_weight_fraction_leaf : float, default=0.0 The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. - max_depth : integer, optional (default=3) + max_depth : int, default=3 maximum depth of the individual regression estimators. The maximum depth limits the number of nodes in the tree. Tune this parameter for best performance; the best value depends on the interaction of the input variables. - min_impurity_decrease : float, optional (default=0.) + min_impurity_decrease : float, default=0.0 A node will be split if this split induces a decrease of the impurity greater than or equal to this value. @@ -2372,30 +1384,34 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): .. versionadded:: 0.19 - min_impurity_split : float, (default=1e-7) + min_impurity_split : float, default=None Threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf. .. deprecated:: 0.19 ``min_impurity_split`` has been deprecated in favor of ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` will change from 1e-7 to 0 in 0.23 and it + ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it will be removed in 0.25. Use ``min_impurity_decrease`` instead. - init : estimator or 'zero', optional (default=None) + init : estimator or 'zero', default=None An estimator object that is used to compute the initial predictions. ``init`` has to provide :term:`fit` and :term:`predict`. If 'zero', the initial raw predictions are set to zero. By default a ``DummyEstimator`` is used, predicting either the average target value (for loss='ls'), or a quantile for the other losses. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - max_features : int, float, string or None, optional (default=None) + random_state : int or RandomState, default=None + Controls the random seed given to each Tree estimator at each + boosting iteration. + In addition, it controls the random permutation of the features at + each split (see Notes for more details). + It also controls the random spliting of the training data to obtain a + validation set if `n_iter_no_change` is not None. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. + + max_features : {'auto', 'sqrt', 'log2'}, int or float, default=None The number of features to consider when looking for the best split: - If int, then consider `max_features` features at each split. @@ -2414,21 +1430,21 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): valid partition of the node samples is found, even if it requires to effectively inspect more than ``max_features`` features. - alpha : float (default=0.9) + alpha : float, default=0.9 The alpha-quantile of the huber loss function and the quantile loss function. Only if ``loss='huber'`` or ``loss='quantile'``. - verbose : int, default: 0 + verbose : int, default=0 Enable verbose output. If 1 then it prints progress and performance once in a while (the more trees the lower the frequency). If greater than 1 then it prints progress and performance for every tree. - max_leaf_nodes : int or None, optional (default=None) + max_leaf_nodes : int, default=None Grow trees with ``max_leaf_nodes`` in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. - warm_start : bool, default: False + warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just erase the previous solution. See :term:`the Glossary `. @@ -2438,14 +1454,14 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): .. deprecated :: 0.22 - validation_fraction : float, optional, default 0.1 + validation_fraction : float, default=0.1 The proportion of training data to set aside as validation set for early stopping. Must be between 0 and 1. Only used if ``n_iter_no_change`` is set to an integer. .. versionadded:: 0.20 - n_iter_no_change : int, default None + n_iter_no_change : int, default=None ``n_iter_no_change`` is used to decide if early stopping will be used to terminate training when validation score is not improving. By default it is set to None to disable early stopping. If set to a @@ -2456,14 +1472,14 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): .. versionadded:: 0.20 - tol : float, optional, default 1e-4 + tol : float, default=1e-4 Tolerance for the early stopping. When the loss is not improving by at least tol for ``n_iter_no_change`` iterations (if set to a number), the training stops. .. versionadded:: 0.20 - ccp_alpha : non-negative float, optional (default=0.0) + ccp_alpha : non-negative float, default=0.0 Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ``ccp_alpha`` will be chosen. By default, no pruning is performed. See @@ -2473,17 +1489,25 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): Attributes ---------- - feature_importances_ : array, shape (n_features,) - The feature importances (the higher, the more important the feature). - - oob_improvement_ : array, shape (n_estimators,) + feature_importances_ : ndarray of shape (n_features,) + The impurity-based feature importances. + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + + oob_improvement_ : ndarray of shape (n_estimators,) The improvement in loss (= deviance) on the out-of-bag samples relative to the previous iteration. ``oob_improvement_[0]`` is the improvement in loss of the first stage over the ``init`` estimator. Only available if ``subsample < 1.0`` - train_score_ : array, shape (n_estimators,) + train_score_ : ndarray of shape (n_estimators,) The i-th score ``train_score_[i]`` is the deviance (= loss) of the model at iteration ``i`` on the in-bag sample. If ``subsample == 1`` this is the deviance on the training data. @@ -2495,9 +1519,15 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): The estimator that provides the initial predictions. Set via the ``init`` argument or ``loss.init_estimator``. - estimators_ : array of DecisionTreeRegressor, shape (n_estimators, 1) + estimators_ : ndarray of DecisionTreeRegressor of shape (n_estimators, 1) The collection of fitted sub-estimators. + n_features_ : int + The number of data features. + + max_features_ : int + The inferred value of max_features. + Notes ----- The features are always randomly permuted at each split. Therefore, @@ -2507,6 +1537,22 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): split. To obtain a deterministic behaviour during fitting, ``random_state`` has to be fixed. + Examples + -------- + >>> from sklearn.datasets import make_regression + >>> from sklearn.ensemble import GradientBoostingRegressor + >>> from sklearn.model_selection import train_test_split + >>> X, y = make_regression(random_state=0) + >>> X_train, X_test, y_train, y_test = train_test_split( + ... X, y, random_state=0) + >>> reg = GradientBoostingRegressor(random_state=0) + >>> reg.fit(X_train, y_train) + GradientBoostingRegressor(random_state=0) + >>> reg.predict(X_test[1:2]) + array([-61...]) + >>> reg.score(X_test, y_test) + 0.4... + See also -------- sklearn.ensemble.HistGradientBoostingRegressor, @@ -2554,14 +1600,14 @@ def predict(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. Returns ------- - y : array, shape (n_samples,) + y : ndarray of shape (n_samples,) The predicted values. """ X = check_array(X, dtype=DTYPE, order="C", accept_sparse='csr') @@ -2576,14 +1622,14 @@ def staged_predict(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. Returns ------- - y : generator of array of shape (n_samples,) + y : generator of ndarray of shape (n_samples,) The predicted value of the input samples. """ for raw_predictions in self._staged_raw_predict(X): @@ -2596,14 +1642,14 @@ def apply(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, its dtype will be converted to ``dtype=np.float32``. If a sparse matrix is provided, it will be converted to a sparse ``csr_matrix``. Returns ------- - X_leaves : array-like, shape (n_samples, n_estimators) + X_leaves : array-like of shape (n_samples, n_estimators) For each datapoint x in X and for each tree in the ensemble, return the index of the leaf x ends up in each estimator. """ diff --git a/sklearn/ensemble/_gb_losses.py b/sklearn/ensemble/_gb_losses.py index 19c66710bf0ad..f400144abc4fc 100644 --- a/sklearn/ensemble/_gb_losses.py +++ b/sklearn/ensemble/_gb_losses.py @@ -46,13 +46,13 @@ def __call__(self, y, raw_predictions, sample_weight=None): Parameters ---------- - y : 1d array, shape (n_samples,) + y : ndarray of shape (n_samples,) True labels. - raw_predictions : 2d array, shape (n_samples, K) + raw_predictions : ndarray of shape (n_samples, K) The raw predictions (i.e. values from the tree leaves). - sample_weight : 1d array, shape (n_samples,), optional + sample_weight : ndarray of shape (n_samples,), default=None Sample weights. """ @@ -62,10 +62,10 @@ def negative_gradient(self, y, raw_predictions, **kargs): Parameters ---------- - y : 1d array, shape (n_samples,) + y : ndarray of shape (n_samples,) The target labels. - raw_predictions : 2d array, shape (n_samples, K) + raw_predictions : ndarray of shape (n_samples, K) The raw predictions (i.e. values from the tree leaves) of the tree ensemble at iteration ``i - 1``. """ @@ -81,18 +81,18 @@ def update_terminal_regions(self, tree, X, y, residual, raw_predictions, ---------- tree : tree.Tree The tree object. - X : 2d array, shape (n, m) + X : ndarray of shape (n_samples, n_features) The data array. - y : 1d array, shape (n,) + y : ndarray of shape (n_samples,) The target labels. - residual : 1d array, shape (n,) + residual : ndarray of shape (n_samples,) The residuals (usually the negative gradient). - raw_predictions : 2d array, shape (n_samples, K) + raw_predictions : ndarray of shape (n_samples, K) The raw predictions (i.e. values from the tree leaves) of the tree ensemble at iteration ``i - 1``. - sample_weight : 1d array, shape (n,) + sample_weight : ndarray of shape (n_samples,) The weight of each sample. - sample_mask : 1d array, shape (n,) + sample_mask : ndarray of shape (n_samples,) The sample mask to be used. learning_rate : float, default=0.1 Learning rate shrinks the contribution of each tree by @@ -129,14 +129,14 @@ def get_init_raw_predictions(self, X, estimator): Parameters ---------- - X : 2d array, shape (n_samples, n_features) + X : ndarray of shape (n_samples, n_features) The data array. - estimator : estimator instance + estimator : object The estimator to use to compute the predictions. Returns ------- - raw_predictions : 2d array, shape (n_samples, K) + raw_predictions : ndarray of shape (n_samples, K) The initial raw predictions. K is equal to 1 for binary classification and regression, and equal to the number of classes for multiclass classification. ``raw_predictions`` is casted @@ -164,7 +164,7 @@ def check_init_estimator(self, estimator): Parameters ---------- - estimator : estimator instance + estimator : object The init estimator to check. """ if not (hasattr(estimator, 'fit') and hasattr(estimator, 'predict')): @@ -196,13 +196,13 @@ def __call__(self, y, raw_predictions, sample_weight=None): Parameters ---------- - y : 1d array, shape (n_samples,) + y : ndarray of shape (n_samples,) True labels. - raw_predictions : 2d array, shape (n_samples, K) - The raw_predictions (i.e. values from the tree leaves). + raw_predictions : ndarray of shape (n_samples, K) + The raw predictions (i.e. values from the tree leaves). - sample_weight : 1d array, shape (n_samples,), optional + sample_weight : ndarray of shape (n_samples,), default=None Sample weights. """ if sample_weight is None: @@ -216,10 +216,10 @@ def negative_gradient(self, y, raw_predictions, **kargs): Parameters ---------- - y : 1d array, shape (n_samples,) + y : ndarray of shape (n_samples,) The target labels. - raw_predictions : 1d array, shape (n_samples,) + raw_predictions : ndarray of shape (n_samples,) The raw predictions (i.e. values from the tree leaves) of the tree ensemble at iteration ``i - 1``. """ @@ -236,18 +236,18 @@ def update_terminal_regions(self, tree, X, y, residual, raw_predictions, ---------- tree : tree.Tree The tree object. - X : 2d array, shape (n, m) + X : ndarray of shape (n_samples, n_features) The data array. - y : 1d array, shape (n,) + y : ndarray of shape (n_samples,) The target labels. - residual : 1d array, shape (n,) + residual : ndarray of shape (n_samples,) The residuals (usually the negative gradient). - raw_predictions : 2d array, shape (n_samples, K) + raw_predictions : ndarray of shape (n_samples, K) The raw predictions (i.e. values from the tree leaves) of the tree ensemble at iteration ``i - 1``. - sample_weight : 1d array, shape (n,) + sample_weight : ndarray of shape (n,) The weight of each sample. - sample_mask : 1d array, shape (n,) + sample_mask : ndarray of shape (n,) The sample mask to be used. learning_rate : float, default=0.1 Learning rate shrinks the contribution of each tree by @@ -279,13 +279,13 @@ def __call__(self, y, raw_predictions, sample_weight=None): Parameters ---------- - y : array, shape (n_samples,) + y : ndarray of shape (n_samples,) True labels. - raw_predictions : array, shape (n_samples, K) - The raw_predictions (i.e. values from the tree leaves). + raw_predictions : ndarray of shape (n_samples, K) + The raw predictions (i.e. values from the tree leaves). - sample_weight : 1d array, shape (n_samples,), optional + sample_weight : ndarray of shape (n_samples,), default=None Sample weights. """ if sample_weight is None: @@ -301,10 +301,10 @@ def negative_gradient(self, y, raw_predictions, **kargs): Parameters ---------- - y : 1d array, shape (n_samples,) + y : ndarray of shape (n_samples,) The target labels. - raw_predictions : array, shape (n_samples, K) + raw_predictions : ndarray of shape (n_samples, K) The raw predictions (i.e. values from the tree leaves) of the tree ensemble at iteration ``i - 1``. """ @@ -327,11 +327,6 @@ class HuberLossFunction(RegressionLossFunction): M-Regression proposed in Friedman 2001. - References - ---------- - J. Friedman, Greedy Function Approximation: A Gradient Boosting - Machine, The Annals of Statistics, Vol. 29, No. 5, 2001. - Parameters ---------- n_classes : int @@ -339,6 +334,11 @@ class HuberLossFunction(RegressionLossFunction): alpha : float, default=0.9 Percentile at which to extract score. + + References + ---------- + J. Friedman, Greedy Function Approximation: A Gradient Boosting + Machine, The Annals of Statistics, Vol. 29, No. 5, 2001. """ def __init__(self, n_classes, alpha=0.9): @@ -354,14 +354,14 @@ def __call__(self, y, raw_predictions, sample_weight=None): Parameters ---------- - y : 1d array, shape (n_samples,) + y : ndarray of shape (n_samples,) True labels. - raw_predictions : 2d array, shape (n_samples, K) + raw_predictions : ndarray of shape (n_samples, K) The raw predictions (i.e. values from the tree leaves) of the tree ensemble. - sample_weight : 1d array, shape (n_samples,), optional + sample_weight : ndarray of shape (n_samples,), default=None Sample weights. """ raw_predictions = raw_predictions.ravel() @@ -394,14 +394,14 @@ def negative_gradient(self, y, raw_predictions, sample_weight=None, Parameters ---------- - y : 1d array, shape (n_samples,) + y : ndarray of shape (n_samples,) The target labels. - raw_predictions : 2d array, shape (n_samples, K) + raw_predictions : ndarray of shape (n_samples, K) The raw predictions (i.e. values from the tree leaves) of the tree ensemble at iteration ``i - 1``. - sample_weight : 1d array, shape (n_samples,), optional + sample_weight : ndarray of shape (n_samples,), default=None Sample weights. """ raw_predictions = raw_predictions.ravel() @@ -443,7 +443,7 @@ class QuantileLossFunction(RegressionLossFunction): n_classes : int Number of classes. - alpha : float, optional (default = 0.9) + alpha : float, default=0.9 The percentile. """ def __init__(self, n_classes, alpha=0.9): @@ -459,14 +459,14 @@ def __call__(self, y, raw_predictions, sample_weight=None): Parameters ---------- - y : 1d array, shape (n_samples,) + y : ndarray of shape (n_samples,) True labels. - raw_predictions : 2d array, shape (n_samples, K) + raw_predictions : ndarray of shape (n_samples, K) The raw predictions (i.e. values from the tree leaves) of the tree ensemble. - sample_weight : 1d array, shape (n_samples,), optional + sample_weight : ndarray of shape (n_samples,), default=None Sample weights. """ raw_predictions = raw_predictions.ravel() @@ -488,11 +488,11 @@ def negative_gradient(self, y, raw_predictions, **kargs): Parameters ---------- - y : 1d array, shape (n_samples,) + y : ndarray of shape (n_samples,) The target labels. - raw_predictions : 2d array, shape (n_samples, K) - The raw_predictions (i.e. values from the tree leaves) of the + raw_predictions : ndarray of shape (n_samples, K) + The raw predictions (i.e. values from the tree leaves) of the tree ensemble at iteration ``i - 1``. """ alpha = self.alpha @@ -519,13 +519,13 @@ def _raw_prediction_to_proba(self, raw_predictions): Parameters ---------- - raw_predictions : 2d array, shape (n_samples, K) + raw_predictions : ndarray of shape (n_samples, K) The raw predictions (i.e. values from the tree leaves) of the tree ensemble. Returns ------- - probas : 2d array, shape (n_samples, K) + probas : ndarray of shape (n_samples, K) The predicted probabilities. """ @@ -535,13 +535,13 @@ def _raw_prediction_to_decision(self, raw_predictions): Parameters ---------- - raw_predictions : 2d array, shape (n_samples, K) + raw_predictions : ndarray of shape (n_samples, K) The raw predictions (i.e. values from the tree leaves) of the tree ensemble. Returns ------- - encoded_predictions : 2d array, shape (n_samples, K) + encoded_predictions : ndarray of shape (n_samples, K) The predicted encoded labels. """ @@ -550,7 +550,7 @@ def check_init_estimator(self, estimator): Parameters ---------- - estimator : estimator instance + estimator : object The init estimator to check. """ if not (hasattr(estimator, 'fit') and @@ -589,14 +589,14 @@ def __call__(self, y, raw_predictions, sample_weight=None): Parameters ---------- - y : 1d array, shape (n_samples,) + y : ndarray of shape (n_samples,) True labels. - raw_predictions : 2d array, shape (n_samples, K) + raw_predictions : ndarray of shape (n_samples, K) The raw predictions (i.e. values from the tree leaves) of the tree ensemble. - sample_weight : 1d array , shape (n_samples,), optional + sample_weight : ndarray of shape (n_samples,), default=None Sample weights. """ # logaddexp(0, v) == log(1.0 + exp(v)) @@ -614,11 +614,11 @@ def negative_gradient(self, y, raw_predictions, **kargs): Parameters ---------- - y : 1d array, shape (n_samples,) + y : ndarray of shape (n_samples,) True labels. - raw_predictions : 2d array, shape (n_samples, K) - The raw_predictions (i.e. values from the tree leaves) of the + raw_predictions : ndarray of shape (n_samples, K) + The raw predictions (i.e. values from the tree leaves) of the tree ensemble at iteration ``i - 1``. """ return y - expit(raw_predictions.ravel()) @@ -696,14 +696,14 @@ def __call__(self, y, raw_predictions, sample_weight=None): Parameters ---------- - y : 1d array, shape (n_samples,) + y : ndarray of shape (n_samples,) True labels. - raw_predictions : 2d array, shape (n_samples, K) + raw_predictions : ndarray of shape (n_samples, K) The raw predictions (i.e. values from the tree leaves) of the tree ensemble. - sample_weight : 1d array, shape (n_samples,), optional + sample_weight : ndarray of shape (n_samples,), default=None Sample weights. """ # create one-hot label encoding @@ -724,14 +724,14 @@ def negative_gradient(self, y, raw_predictions, k=0, **kwargs): Parameters ---------- - y : 1d array, shape (n_samples,) + y : ndarray of shape (n_samples,) The target labels. - raw_predictions : 2d array, shape (n_samples, K) - The raw_predictions (i.e. values from the tree leaves) of the + raw_predictions : ndarray of shape (n_samples, K) + The raw predictions (i.e. values from the tree leaves) of the tree ensemble at iteration ``i - 1``. - k : int, optional default=0 + k : int, default=0 The index of the class. """ return y - np.nan_to_num(np.exp(raw_predictions[:, k] - @@ -779,14 +779,14 @@ class ExponentialLoss(ClassificationLossFunction): Same loss as AdaBoost. - References - ---------- - Greg Ridgeway, Generalized Boosted Models: A guide to the gbm package, 2007 - Parameters ---------- n_classes : int Number of classes. + + References + ---------- + Greg Ridgeway, Generalized Boosted Models: A guide to the gbm package, 2007 """ def __init__(self, n_classes): if n_classes != 2: @@ -803,14 +803,14 @@ def __call__(self, y, raw_predictions, sample_weight=None): Parameters ---------- - y : 1d array, shape (n_samples,) + y : ndarray of shape (n_samples,) True labels. - raw_predictions : 2d array, shape (n_samples, K) + raw_predictions : ndarray of shape (n_samples, K) The raw predictions (i.e. values from the tree leaves) of the tree ensemble. - sample_weight : 1d array, shape (n_samples,), optional + sample_weight : ndarray of shape (n_samples,), default=None Sample weights. """ raw_predictions = raw_predictions.ravel() @@ -825,10 +825,10 @@ def negative_gradient(self, y, raw_predictions, **kargs): Parameters ---------- - y : 1d array, shape (n_samples,) + y : ndarray of shape (n_samples,) True labels. - raw_predictions : 2d array, shape (n_samples, K) + raw_predictions : ndarray of shape (n_samples, K) The raw predictions (i.e. values from the tree leaves) of the tree ensemble at iteration ``i - 1``. """ diff --git a/sklearn/ensemble/_hist_gradient_boosting/_loss.pyx b/sklearn/ensemble/_hist_gradient_boosting/_loss.pyx index 418a9124d37fa..821a81a48fcf3 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/_loss.pyx +++ b/sklearn/ensemble/_hist_gradient_boosting/_loss.pyx @@ -27,9 +27,51 @@ def _update_gradients_least_squares( n_samples = raw_predictions.shape[0] for i in prange(n_samples, schedule='static', nogil=True): + # Note: a more correct exp is 2 * (raw_predictions - y_true) + # but since we use 1 for the constant hessian value (and not 2) this + # is strictly equivalent for the leaves values. gradients[i] = raw_predictions[i] - y_true[i] +def _update_gradients_hessians_least_squares( + G_H_DTYPE_C [::1] gradients, # OUT + G_H_DTYPE_C [::1] hessians, # OUT + const Y_DTYPE_C [::1] y_true, # IN + const Y_DTYPE_C [::1] raw_predictions, # IN + const Y_DTYPE_C [::1] sample_weight): # IN + + cdef: + int n_samples + int i + + n_samples = raw_predictions.shape[0] + for i in prange(n_samples, schedule='static', nogil=True): + # Note: a more correct exp is 2 * (raw_predictions - y_true) * sample_weight + # but since we use 1 for the constant hessian value (and not 2) this + # is strictly equivalent for the leaves values. + gradients[i] = (raw_predictions[i] - y_true[i]) * sample_weight[i] + hessians[i] = sample_weight[i] + + +def _update_gradients_hessians_least_absolute_deviation( + G_H_DTYPE_C [::1] gradients, # OUT + G_H_DTYPE_C [::1] hessians, # OUT + const Y_DTYPE_C [::1] y_true, # IN + const Y_DTYPE_C [::1] raw_predictions, # IN + const Y_DTYPE_C [::1] sample_weight): # IN + + cdef: + int n_samples + int i + + n_samples = raw_predictions.shape[0] + for i in prange(n_samples, schedule='static', nogil=True): + # gradient = sign(raw_predicition - y_pred) * sample_weight + gradients[i] = sample_weight[i] * (2 * + (y_true[i] - raw_predictions[i] < 0) - 1) + hessians[i] = sample_weight[i] + + def _update_gradients_least_absolute_deviation( G_H_DTYPE_C [::1] gradients, # OUT const Y_DTYPE_C [::1] y_true, # IN @@ -49,44 +91,66 @@ def _update_gradients_hessians_binary_crossentropy( G_H_DTYPE_C [::1] gradients, # OUT G_H_DTYPE_C [::1] hessians, # OUT const Y_DTYPE_C [::1] y_true, # IN - const Y_DTYPE_C [::1] raw_predictions): # IN + const Y_DTYPE_C [::1] raw_predictions, # IN + const Y_DTYPE_C [::1] sample_weight): # IN cdef: int n_samples Y_DTYPE_C p_i # proba that ith sample belongs to positive class int i n_samples = raw_predictions.shape[0] - for i in prange(n_samples, schedule='static', nogil=True): - p_i = _cexpit(raw_predictions[i]) - gradients[i] = p_i - y_true[i] - hessians[i] = p_i * (1. - p_i) + if sample_weight is None: + for i in prange(n_samples, schedule='static', nogil=True): + p_i = _cexpit(raw_predictions[i]) + gradients[i] = p_i - y_true[i] + hessians[i] = p_i * (1. - p_i) + else: + for i in prange(n_samples, schedule='static', nogil=True): + p_i = _cexpit(raw_predictions[i]) + gradients[i] = (p_i - y_true[i]) * sample_weight[i] + hessians[i] = p_i * (1. - p_i) * sample_weight[i] def _update_gradients_hessians_categorical_crossentropy( G_H_DTYPE_C [:, ::1] gradients, # OUT G_H_DTYPE_C [:, ::1] hessians, # OUT const Y_DTYPE_C [::1] y_true, # IN - const Y_DTYPE_C [:, ::1] raw_predictions): # IN + const Y_DTYPE_C [:, ::1] raw_predictions, # IN + const Y_DTYPE_C [::1] sample_weight): # IN cdef: int prediction_dim = raw_predictions.shape[0] int n_samples = raw_predictions.shape[1] int k # class index int i # sample index + Y_DTYPE_C sw # p[i, k] is the probability that class(ith sample) == k. # It's the softmax of the raw predictions Y_DTYPE_C [:, ::1] p = np.empty(shape=(n_samples, prediction_dim)) Y_DTYPE_C p_i_k - for i in prange(n_samples, schedule='static', nogil=True): - # first compute softmaxes of sample i for each class - for k in range(prediction_dim): - p[i, k] = raw_predictions[k, i] # prepare softmax - _compute_softmax(p, i) - # then update gradients and hessians - for k in range(prediction_dim): - p_i_k = p[i, k] - gradients[k, i] = p_i_k - (y_true[i] == k) - hessians[k, i] = p_i_k * (1. - p_i_k) + if sample_weight is None: + for i in prange(n_samples, schedule='static', nogil=True): + # first compute softmaxes of sample i for each class + for k in range(prediction_dim): + p[i, k] = raw_predictions[k, i] # prepare softmax + _compute_softmax(p, i) + # then update gradients and hessians + for k in range(prediction_dim): + p_i_k = p[i, k] + gradients[k, i] = p_i_k - (y_true[i] == k) + hessians[k, i] = p_i_k * (1. - p_i_k) + else: + for i in prange(n_samples, schedule='static', nogil=True): + # first compute softmaxes of sample i for each class + for k in range(prediction_dim): + p[i, k] = raw_predictions[k, i] # prepare softmax + _compute_softmax(p, i) + # then update gradients and hessians + sw = sample_weight[i] + for k in range(prediction_dim): + p_i_k = p[i, k] + gradients[k, i] = (p_i_k - (y_true[i] == k)) * sw + hessians[k, i] = (p_i_k * (1. - p_i_k)) * sw cdef inline void _compute_softmax(Y_DTYPE_C [:, ::1] p, const int i) nogil: diff --git a/sklearn/ensemble/_hist_gradient_boosting/binning.py b/sklearn/ensemble/_hist_gradient_boosting/binning.py index 18cddca2d867f..83c338d89633e 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/binning.py +++ b/sklearn/ensemble/_hist_gradient_boosting/binning.py @@ -32,11 +32,13 @@ def _find_binning_thresholds(data, max_bins, subsample, random_state): instead of the quantiles. subsample : int or None If ``n_samples > subsample``, then ``sub_samples`` samples will be - randomly choosen to compute the quantiles. If ``None``, the whole data + randomly chosen to compute the quantiles. If ``None``, the whole data is used. - random_state: int or numpy.random.RandomState or None + random_state: int, RandomState instance or None Pseudo-random number generator to control the random sub-sampling. - See :term:`random_state`. + Pass an int for reproducible output across multiple + function calls. + See :term: `Glossary `. Return ------ @@ -107,12 +109,13 @@ class _BinMapper(TransformerMixin, BaseEstimator): instead of the quantiles. subsample : int or None, optional (default=2e5) If ``n_samples > subsample``, then ``sub_samples`` samples will be - randomly choosen to compute the quantiles. If ``None``, the whole data + randomly chosen to compute the quantiles. If ``None``, the whole data is used. - random_state: int or numpy.random.RandomState or None, \ - optional (default=None) + random_state: int, RandomState instance or None Pseudo-random number generator to control the random sub-sampling. - See :term:`random_state`. + Pass an int for reproducible output across multiple + function calls. + See :term: `Glossary `. Attributes ---------- @@ -126,7 +129,7 @@ class _BinMapper(TransformerMixin, BaseEstimator): equal to ``n_bins - 1``. missing_values_bin_idx_ : uint8 The index of the bin where missing values are mapped. This is a - constant accross all features. This corresponds to the last bin, and + constant across all features. This corresponds to the last bin, and it is always equal to ``n_bins - 1``. Note that if ``n_bins_missing_`` is less than ``n_bins - 1`` for a given feature, then there are empty (and unused) bins. diff --git a/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py b/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py index 7310ab95b224f..5db39f07c7ce1 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py +++ b/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py @@ -9,7 +9,8 @@ from ...base import (BaseEstimator, RegressorMixin, ClassifierMixin, is_classifier) from ...utils import check_X_y, check_random_state, check_array, resample -from ...utils.validation import check_is_fitted +from ...utils.validation import (check_is_fitted, + check_consistent_length, _check_sample_weight) from ...utils.multiclass import check_classification_targets from ...metrics import check_scoring from ...model_selection import train_test_split @@ -28,8 +29,8 @@ class BaseHistGradientBoosting(BaseEstimator, ABC): @abstractmethod def __init__(self, loss, learning_rate, max_iter, max_leaf_nodes, max_depth, min_samples_leaf, l2_regularization, max_bins, - warm_start, scoring, validation_fraction, n_iter_no_change, - tol, verbose, random_state): + warm_start, early_stopping, scoring, validation_fraction, + n_iter_no_change, tol, verbose, random_state): self.loss = loss self.learning_rate = learning_rate self.max_iter = max_iter @@ -39,6 +40,7 @@ def __init__(self, loss, learning_rate, max_iter, max_leaf_nodes, self.l2_regularization = l2_regularization self.max_bins = max_bins self.warm_start = warm_start + self.early_stopping = early_stopping self.scoring = scoring self.validation_fraction = validation_fraction self.n_iter_no_change = n_iter_no_change @@ -64,7 +66,7 @@ def _validate_parameters(self): if self.max_iter < 1: raise ValueError('max_iter={} must not be smaller ' 'than 1.'.format(self.max_iter)) - if self.n_iter_no_change is not None and self.n_iter_no_change < 0: + if self.n_iter_no_change < 0: raise ValueError('n_iter_no_change={} must be ' 'positive.'.format(self.n_iter_no_change)) if (self.validation_fraction is not None and @@ -80,7 +82,7 @@ def _validate_parameters(self): raise ValueError('max_bins={} should be no smaller than 2 ' 'and no larger than 255.'.format(self.max_bins)) - def fit(self, X, y): + def fit(self, X, y, sample_weight=None): """Fit the gradient boosting model. Parameters @@ -91,6 +93,9 @@ def fit(self, X, y): y : array-like of shape (n_samples,) Target values. + sample_weight : array-like of shape (n_samples,) default=None + Weights of training data. + Returns ------- self : object @@ -101,8 +106,17 @@ def fit(self, X, y): acc_compute_hist_time = 0. # time spent computing histograms # time spent predicting X for gradient and hessians update acc_prediction_time = 0. - X, y = check_X_y(X, y, dtype=[X_DTYPE], force_all_finite=False) + X, y = self._validate_data(X, y, dtype=[X_DTYPE], + force_all_finite=False) y = self._encode_y(y) + check_consistent_length(X, y) + # Do not create unit sample weights by default to later skip some + # computation + if sample_weight is not None: + sample_weight = _check_sample_weight(sample_weight, X, + dtype=np.float64) + # TODO: remove when PDP suports sample weights + self._fitted_with_sw = True rng = check_random_state(self.random_state) @@ -114,7 +128,7 @@ def fit(self, X, y): dtype='u8') self._validate_parameters() - self.n_features_ = X.shape[1] # used for validation in predict() + n_samples, self.n_features_ = X.shape # used for validation in predict # we need this stateful variable to tell raw_predict() that it was # called from fit() (this current method), and that the data it has @@ -126,10 +140,11 @@ def fit(self, X, y): # data. self._in_fit = True - self.loss_ = self._get_loss() - - self.do_early_stopping_ = (self.n_iter_no_change is not None and - self.n_iter_no_change > 0) + self.loss_ = self._get_loss(sample_weight=sample_weight) + if self.early_stopping == 'auto': + self.do_early_stopping_ = n_samples > 10000 + else: + self.do_early_stopping_ = self.early_stopping # create validation data if needed self._use_validation_data = self.validation_fraction is not None @@ -141,12 +156,23 @@ def fit(self, X, y): # This is needed in order to have the same split when using # warm starting. - X_train, X_val, y_train, y_val = train_test_split( - X, y, test_size=self.validation_fraction, stratify=stratify, - random_state=self._random_seed) + if sample_weight is None: + X_train, X_val, y_train, y_val = train_test_split( + X, y, test_size=self.validation_fraction, + stratify=stratify, + random_state=self._random_seed) + sample_weight_train = sample_weight_val = None + else: + # TODO: incorporate sample_weight in sampling here, as well as + # stratify + (X_train, X_val, y_train, y_val, sample_weight_train, + sample_weight_val) = train_test_split( + X, y, sample_weight, test_size=self.validation_fraction, + stratify=stratify, + random_state=self._random_seed) else: - X_train, y_train = X, y - X_val, y_val = None, None + X_train, y_train, sample_weight_train = X, y, sample_weight + X_val = y_val = sample_weight_val = None has_missing_values = np.isnan(X_train).any(axis=0).astype(np.uint8) @@ -183,7 +209,7 @@ def fit(self, X, y): # n_trees_per_iterations is n_classes in multiclass classification, # else 1. self._baseline_prediction = self.loss_.get_baseline_prediction( - y_train, self.n_trees_per_iteration_ + y_train, sample_weight_train, self.n_trees_per_iteration_ ) raw_predictions = np.zeros( shape=(self.n_trees_per_iteration_, n_samples), @@ -195,7 +221,8 @@ def fit(self, X, y): # shape = (n_trees_per_iteration, n_samples). gradients, hessians = self.loss_.init_gradients_and_hessians( n_samples=n_samples, - prediction_dim=self.n_trees_per_iteration_ + prediction_dim=self.n_trees_per_iteration_, + sample_weight=sample_weight_train ) # predictors is a matrix (list of lists) of TreePredictor objects @@ -232,7 +259,9 @@ def fit(self, X, y): raw_predictions_val += self._baseline_prediction self._check_early_stopping_loss(raw_predictions, y_train, - raw_predictions_val, y_val) + sample_weight_train, + raw_predictions_val, y_val, + sample_weight_val) else: self.scorer_ = check_scoring(self, self.scoring) # scorer_ is a callable with signature (est, X, y) and @@ -244,12 +273,15 @@ def fit(self, X, y): # Compute the subsample set (X_binned_small_train, - y_small_train) = self._get_small_trainset( - X_binned_train, y_train, self._random_seed) + y_small_train, + sample_weight_small_train) = self._get_small_trainset( + X_binned_train, y_train, sample_weight_train, + self._random_seed) self._check_early_stopping_scorer( X_binned_small_train, y_small_train, - X_binned_val, y_val, + sample_weight_small_train, + X_binned_val, y_val, sample_weight_val, ) begin_at_stage = 0 @@ -270,15 +302,21 @@ def fit(self, X, y): # Compute raw predictions raw_predictions = self._raw_predict(X_binned_train) + if self.do_early_stopping_ and self._use_validation_data: + raw_predictions_val = self._raw_predict(X_binned_val) if self.do_early_stopping_ and self.scoring != 'loss': # Compute the subsample set - X_binned_small_train, y_small_train = self._get_small_trainset( - X_binned_train, y_train, self._random_seed) + (X_binned_small_train, + y_small_train, + sample_weight_small_train) = self._get_small_trainset( + X_binned_train, y_train, sample_weight_train, + self._random_seed) # Initialize the gradients and hessians gradients, hessians = self.loss_.init_gradients_and_hessians( n_samples=n_samples, + sample_weight=sample_weight_train, prediction_dim=self.n_trees_per_iteration_ ) @@ -287,6 +325,14 @@ def fit(self, X, y): begin_at_stage = self.n_iter_ + # initialize gradients and hessians (empty arrays). + # shape = (n_trees_per_iteration, n_samples). + gradients, hessians = self.loss_.init_gradients_and_hessians( + n_samples=n_samples, + prediction_dim=self.n_trees_per_iteration_, + sample_weight=sample_weight_train + ) + for iteration in range(begin_at_stage, self.max_iter): if self.verbose: @@ -296,7 +342,8 @@ def fit(self, X, y): # Update gradients and hessians, inplace self.loss_.update_gradients_and_hessians(gradients, hessians, - y_train, raw_predictions) + y_train, raw_predictions, + sample_weight_train) # Append a list since there may be more than 1 predictor per iter predictors.append([]) @@ -322,7 +369,8 @@ def fit(self, X, y): if self.loss_.need_update_leaves_values: self.loss_.update_leaves_values(grower, y_train, - raw_predictions[k, :]) + raw_predictions[k, :], + sample_weight_train) predictor = grower.make_predictor( bin_thresholds=self.bin_mapper_.bin_thresholds_ @@ -348,16 +396,19 @@ def fit(self, X, y): self.bin_mapper_.missing_values_bin_idx_ ) ) + else: + raw_predictions_val = None should_early_stop = self._check_early_stopping_loss( - raw_predictions, y_train, - raw_predictions_val, y_val + raw_predictions, y_train, sample_weight_train, + raw_predictions_val, y_val, sample_weight_val ) else: should_early_stop = self._check_early_stopping_scorer( X_binned_small_train, y_small_train, - X_binned_val, y_val, + sample_weight_small_train, + X_binned_val, y_val, sample_weight_val ) if self.verbose: @@ -402,12 +453,14 @@ def _clear_state(self): if hasattr(self, var): delattr(self, var) - def _get_small_trainset(self, X_binned_train, y_train, seed): + def _get_small_trainset(self, X_binned_train, y_train, sample_weight_train, + seed): """Compute the indices of the subsample set and return this set. For efficiency, we need to subsample the training set to compute scores with scorers. """ + # TODO: incorporate sample_weights here in `resample` subsample_size = 10000 if X_binned_train.shape[0] > subsample_size: indices = np.arange(X_binned_train.shape[0]) @@ -417,29 +470,48 @@ def _get_small_trainset(self, X_binned_train, y_train, seed): stratify=stratify) X_binned_small_train = X_binned_train[indices] y_small_train = y_train[indices] + if sample_weight_train is not None: + sample_weight_small_train = sample_weight_train[indices] + else: + sample_weight_small_train = None X_binned_small_train = np.ascontiguousarray(X_binned_small_train) - return X_binned_small_train, y_small_train + return (X_binned_small_train, y_small_train, + sample_weight_small_train) else: - return X_binned_train, y_train + return X_binned_train, y_train, sample_weight_train def _check_early_stopping_scorer(self, X_binned_small_train, y_small_train, - X_binned_val, y_val): + sample_weight_small_train, + X_binned_val, y_val, sample_weight_val): """Check if fitting should be early-stopped based on scorer. Scores are computed on validation data or on training data. """ if is_classifier(self): y_small_train = self.classes_[y_small_train.astype(int)] - self.train_score_.append( - self.scorer_(self, X_binned_small_train, y_small_train) - ) + + if sample_weight_small_train is None: + self.train_score_.append( + self.scorer_(self, X_binned_small_train, y_small_train) + ) + else: + self.train_score_.append( + self.scorer_(self, X_binned_small_train, y_small_train, + sample_weight=sample_weight_small_train) + ) if self._use_validation_data: if is_classifier(self): y_val = self.classes_[y_val.astype(int)] - self.validation_score_.append( - self.scorer_(self, X_binned_val, y_val) - ) + if sample_weight_val is None: + self.validation_score_.append( + self.scorer_(self, X_binned_val, y_val) + ) + else: + self.validation_score_.append( + self.scorer_(self, X_binned_val, y_val, + sample_weight=sample_weight_val) + ) return self._should_stop(self.validation_score_) else: return self._should_stop(self.train_score_) @@ -447,20 +519,22 @@ def _check_early_stopping_scorer(self, X_binned_small_train, y_small_train, def _check_early_stopping_loss(self, raw_predictions, y_train, + sample_weight_train, raw_predictions_val, - y_val): + y_val, + sample_weight_val): """Check if fitting should be early-stopped based on loss. Scores are computed on validation data or on training data. """ self.train_score_.append( - -self.loss_(y_train, raw_predictions) + -self.loss_(y_train, raw_predictions, sample_weight_train) ) if self._use_validation_data: self.validation_score_.append( - -self.loss_(y_val, raw_predictions_val) + -self.loss_(y_val, raw_predictions_val, sample_weight_val) ) return self._should_stop(self.validation_score_) else: @@ -611,6 +685,13 @@ def _compute_partial_dependence_recursion(self, grid, target_features): (n_trees_per_iteration, n_samples) The value of the partial dependence function on each grid point. """ + + if getattr(self, '_fitted_with_sw', False): + raise NotImplementedError("{} does not support partial dependence " + "plots with the 'recursion' method when " + "sample weights were given during fit " + "time.".format(self.__class__.__name__)) + grid = np.asarray(grid, dtype=X_DTYPE, order='C') averaged_predictions = np.zeros( (self.n_trees_per_iteration_, grid.shape[0]), dtype=Y_DTYPE) @@ -628,7 +709,7 @@ def _more_tags(self): return {'allow_nan': True} @abstractmethod - def _get_loss(self): + def _get_loss(self, sample_weight): pass @abstractmethod @@ -693,8 +774,8 @@ class HistGradientBoostingRegressor(RegressorMixin, BaseHistGradientBoosting): than 1. If None, there is no maximum limit. max_depth : int or None, optional (default=None) The maximum depth of each tree. The depth of a tree is the number of - nodes to go from the root to the deepest leaf. Must be strictly greater - than 1. Depth isn't constrained by default. + edges to go from the root to the deepest leaf. + Depth isn't constrained by default. min_samples_leaf : int, optional (default=20) The minimum number of samples per leaf. For small datasets with less than a few hundred samples, it is recommended to lower this value @@ -714,21 +795,25 @@ class HistGradientBoostingRegressor(RegressorMixin, BaseHistGradientBoosting): and add more estimators to the ensemble. For results to be valid, the estimator should be re-trained on the same data only. See :term:`the Glossary `. - scoring : str or callable or None, optional (default=None) + early_stopping : 'auto' or bool (default='auto') + If 'auto', early stopping is enabled if the sample size is larger than + 10000. If True, early stopping is enabled, otherwise early stopping is + disabled. + scoring : str or callable or None, optional (default='loss') Scoring parameter to use for early stopping. It can be a single string (see :ref:`scoring_parameter`) or a callable (see :ref:`scoring`). If None, the estimator's default scorer is used. If ``scoring='loss'``, early stopping is checked w.r.t the loss value. - Only used if ``n_iter_no_change`` is not None. + Only used if early stopping is performed. validation_fraction : int or float or None, optional (default=0.1) Proportion (or absolute size) of training data to set aside as validation data for early stopping. If None, early stopping is done on - the training data. Only used if ``n_iter_no_change`` is not None. - n_iter_no_change : int or None, optional (default=None) + the training data. Only used if early stopping is performed. + n_iter_no_change : int, optional (default=10) Used to determine when to "early stop". The fitting process is stopped when none of the last ``n_iter_no_change`` scores are better than the ``n_iter_no_change - 1`` -th-to-last one, up to some - tolerance. If None or 0, no early-stopping is done. + tolerance. Only used if early stopping is performed. tol : float or None, optional (default=1e-7) The absolute tolerance to use when comparing scores during early stopping. The higher the tolerance, the more likely we are to early @@ -741,13 +826,15 @@ class HistGradientBoostingRegressor(RegressorMixin, BaseHistGradientBoosting): optional (default=None) Pseudo-random number generator to control the subsampling in the binning process, and the train/validation data split if early stopping - is enabled. See :term:`random_state`. + is enabled. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Attributes ---------- n_iter_ : int - The number of iterations as selected by early stopping (if - n_iter_no_change is not None). Otherwise it corresponds to max_iter. + The number of iterations as selected by early stopping, depending on + the `early_stopping` parameter. Otherwise it corresponds to max_iter. n_trees_per_iteration_ : int The number of tree that are built at each iteration. For regressors, this is always 1. @@ -780,16 +867,16 @@ class HistGradientBoostingRegressor(RegressorMixin, BaseHistGradientBoosting): def __init__(self, loss='least_squares', learning_rate=0.1, max_iter=100, max_leaf_nodes=31, max_depth=None, min_samples_leaf=20, l2_regularization=0., max_bins=255, - warm_start=False, scoring=None, validation_fraction=0.1, - n_iter_no_change=None, tol=1e-7, verbose=0, - random_state=None): + warm_start=False, early_stopping='auto', scoring='loss', + validation_fraction=0.1, n_iter_no_change=10, tol=1e-7, + verbose=0, random_state=None): super(HistGradientBoostingRegressor, self).__init__( loss=loss, learning_rate=learning_rate, max_iter=max_iter, max_leaf_nodes=max_leaf_nodes, max_depth=max_depth, min_samples_leaf=min_samples_leaf, l2_regularization=l2_regularization, max_bins=max_bins, - warm_start=warm_start, scoring=scoring, - validation_fraction=validation_fraction, + warm_start=warm_start, early_stopping=early_stopping, + scoring=scoring, validation_fraction=validation_fraction, n_iter_no_change=n_iter_no_change, tol=tol, verbose=verbose, random_state=random_state) @@ -816,8 +903,8 @@ def _encode_y(self, y): y = y.astype(Y_DTYPE, copy=False) return y - def _get_loss(self): - return _LOSSES[self.loss]() + def _get_loss(self, sample_weight): + return _LOSSES[self.loss](sample_weight=sample_weight) class HistGradientBoostingClassifier(BaseHistGradientBoosting, @@ -876,8 +963,8 @@ class HistGradientBoostingClassifier(BaseHistGradientBoosting, than 1. If None, there is no maximum limit. max_depth : int or None, optional (default=None) The maximum depth of each tree. The depth of a tree is the number of - nodes to go from the root to the deepest leaf. Must be strictly greater - than 1. Depth isn't constrained by default. + edges to go from the root to the deepest leaf. + Depth isn't constrained by default. min_samples_leaf : int, optional (default=20) The minimum number of samples per leaf. For small datasets with less than a few hundred samples, it is recommended to lower this value @@ -896,21 +983,25 @@ class HistGradientBoostingClassifier(BaseHistGradientBoosting, and add more estimators to the ensemble. For results to be valid, the estimator should be re-trained on the same data only. See :term:`the Glossary `. - scoring : str or callable or None, optional (default=None) + early_stopping : 'auto' or bool (default='auto') + If 'auto', early stopping is enabled if the sample size is larger than + 10000. If True, early stopping is enabled, otherwise early stopping is + disabled. + scoring : str or callable or None, optional (default='loss') Scoring parameter to use for early stopping. It can be a single string (see :ref:`scoring_parameter`) or a callable (see :ref:`scoring`). If None, the estimator's default scorer is used. If ``scoring='loss'``, early stopping is checked - w.r.t the loss value. Only used if ``n_iter_no_change`` is not None. + w.r.t the loss value. Only used if early stopping is performed. validation_fraction : int or float or None, optional (default=0.1) Proportion (or absolute size) of training data to set aside as validation data for early stopping. If None, early stopping is done on - the training data. - n_iter_no_change : int or None, optional (default=None) + the training data. Only used if early stopping is performed. + n_iter_no_change : int, optional (default=10) Used to determine when to "early stop". The fitting process is stopped when none of the last ``n_iter_no_change`` scores are better than the ``n_iter_no_change - 1`` -th-to-last one, up to some - tolerance. If None or 0, no early-stopping is done. + tolerance. Only used if early stopping is performed. tol : float or None, optional (default=1e-7) The absolute tolerance to use when comparing scores. The higher the tolerance, the more likely we are to early stop: higher tolerance @@ -923,13 +1014,17 @@ class HistGradientBoostingClassifier(BaseHistGradientBoosting, optional (default=None) Pseudo-random number generator to control the subsampling in the binning process, and the train/validation data split if early stopping - is enabled. See :term:`random_state`. + is enabled. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Attributes ---------- + classes_ : array, shape = (n_classes,) + Class labels. n_iter_ : int - The number of estimators as selected by early stopping (if - n_iter_no_change is not None). Otherwise it corresponds to max_iter. + The number of iterations as selected by early stopping, depending on + the `early_stopping` parameter. Otherwise it corresponds to max_iter. n_trees_per_iteration_ : int The number of tree that are built at each iteration. This is equal to 1 for binary classification, and to ``n_classes`` for multiclass @@ -950,7 +1045,7 @@ class HistGradientBoostingClassifier(BaseHistGradientBoosting, -------- >>> # To use this experimental feature, we need to explicitly ask for it: >>> from sklearn.experimental import enable_hist_gradient_boosting # noqa - >>> from sklearn.ensemble import HistGradientBoostingRegressor + >>> from sklearn.ensemble import HistGradientBoostingClassifier >>> from sklearn.datasets import load_iris >>> X, y = load_iris(return_X_y=True) >>> clf = HistGradientBoostingClassifier().fit(X, y) @@ -964,15 +1059,16 @@ class HistGradientBoostingClassifier(BaseHistGradientBoosting, def __init__(self, loss='auto', learning_rate=0.1, max_iter=100, max_leaf_nodes=31, max_depth=None, min_samples_leaf=20, l2_regularization=0., max_bins=255, warm_start=False, - scoring=None, validation_fraction=0.1, n_iter_no_change=None, - tol=1e-7, verbose=0, random_state=None): + early_stopping='auto', scoring='loss', + validation_fraction=0.1, n_iter_no_change=10, tol=1e-7, + verbose=0, random_state=None): super(HistGradientBoostingClassifier, self).__init__( loss=loss, learning_rate=learning_rate, max_iter=max_iter, max_leaf_nodes=max_leaf_nodes, max_depth=max_depth, min_samples_leaf=min_samples_leaf, l2_regularization=l2_regularization, max_bins=max_bins, - warm_start=warm_start, scoring=scoring, - validation_fraction=validation_fraction, + warm_start=warm_start, early_stopping=early_stopping, + scoring=scoring, validation_fraction=validation_fraction, n_iter_no_change=n_iter_no_change, tol=tol, verbose=verbose, random_state=random_state) @@ -1045,7 +1141,7 @@ def _encode_y(self, y): encoded_y = encoded_y.astype(Y_DTYPE, copy=False) return encoded_y - def _get_loss(self): + def _get_loss(self, sample_weight): if (self.loss == 'categorical_crossentropy' and self.n_trees_per_iteration_ == 1): raise ValueError("'categorical_crossentropy' is not suitable for " @@ -1054,8 +1150,10 @@ def _get_loss(self): if self.loss == 'auto': if self.n_trees_per_iteration_ == 1: - return _LOSSES['binary_crossentropy']() + return _LOSSES['binary_crossentropy']( + sample_weight=sample_weight) else: - return _LOSSES['categorical_crossentropy']() + return _LOSSES['categorical_crossentropy']( + sample_weight=sample_weight) - return _LOSSES[self.loss]() + return _LOSSES[self.loss](sample_weight=sample_weight) diff --git a/sklearn/ensemble/_hist_gradient_boosting/grower.py b/sklearn/ensemble/_hist_gradient_boosting/grower.py index c7d303b8f6201..bbee8f6c4585c 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/grower.py +++ b/sklearn/ensemble/_hist_gradient_boosting/grower.py @@ -135,7 +135,8 @@ class TreeGrower: maximum limit. max_depth : int or None, optional (default=None) The maximum depth of each tree. The depth of a tree is the number of - nodes to go from the root to the deepest leaf. + edges to go from the root to the deepest leaf. + Depth isn't constrained by default. min_samples_leaf : int, optional (default=20) The minimum number of samples per leaf. min_gain_to_split : float, optional (default=0.) @@ -230,9 +231,9 @@ def _validate_parameters(self, X_binned, max_leaf_nodes, max_depth, if max_leaf_nodes is not None and max_leaf_nodes <= 1: raise ValueError('max_leaf_nodes={} should not be' ' smaller than 2'.format(max_leaf_nodes)) - if max_depth is not None and max_depth <= 1: + if max_depth is not None and max_depth < 1: raise ValueError('max_depth={} should not be' - ' smaller than 2'.format(max_depth)) + ' smaller than 1'.format(max_depth)) if min_samples_leaf < 1: raise ValueError('min_samples_leaf={} should ' 'not be smaller than 1'.format(min_samples_leaf)) @@ -354,16 +355,16 @@ def split_next(self): self.n_nodes += 2 - if self.max_depth is not None and depth == self.max_depth: + if (self.max_leaf_nodes is not None + and n_leaf_nodes == self.max_leaf_nodes): self._finalize_leaf(left_child_node) self._finalize_leaf(right_child_node) + self._finalize_splittable_nodes() return left_child_node, right_child_node - if (self.max_leaf_nodes is not None - and n_leaf_nodes == self.max_leaf_nodes): + if self.max_depth is not None and depth == self.max_depth: self._finalize_leaf(left_child_node) self._finalize_leaf(right_child_node) - self._finalize_splittable_nodes() return left_child_node, right_child_node if left_child_node.n_samples < self.min_samples_leaf * 2: diff --git a/sklearn/ensemble/_hist_gradient_boosting/loss.py b/sklearn/ensemble/_hist_gradient_boosting/loss.py index bcfec023b5571..2dbf8bd58773e 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/loss.py +++ b/sklearn/ensemble/_hist_gradient_boosting/loss.py @@ -18,13 +18,27 @@ from .common import Y_DTYPE from .common import G_H_DTYPE from ._loss import _update_gradients_least_squares +from ._loss import _update_gradients_hessians_least_squares from ._loss import _update_gradients_least_absolute_deviation +from ._loss import _update_gradients_hessians_least_absolute_deviation from ._loss import _update_gradients_hessians_binary_crossentropy from ._loss import _update_gradients_hessians_categorical_crossentropy +from ...utils.stats import _weighted_percentile class BaseLoss(ABC): """Base class for a loss.""" + def __init__(self, hessians_are_constant): + self.hessians_are_constant = hessians_are_constant + + def __call__(self, y_true, raw_predictions, sample_weight): + """Return the weighted average loss""" + return np.average(self.pointwise_loss(y_true, raw_predictions), + weights=sample_weight) + + @abstractmethod + def pointwise_loss(self, y_true, raw_predictions): + """Return loss value for each input""" # This variable indicates whether the loss requires the leaves values to # be updated once the tree has been trained. The trees are trained to @@ -36,7 +50,8 @@ class BaseLoss(ABC): # (https://statweb.stanford.edu/~jhf/ftp/trebst.pdf) for the theory. need_update_leaves_values = False - def init_gradients_and_hessians(self, n_samples, prediction_dim): + def init_gradients_and_hessians(self, n_samples, prediction_dim, + sample_weight): """Return initial gradients and hessians. Unless hessians are constant, arrays are initialized with undefined @@ -46,12 +61,16 @@ def init_gradients_and_hessians(self, n_samples, prediction_dim): ---------- n_samples : int The number of samples passed to `fit()`. + prediction_dim : int The dimension of a raw prediction, i.e. the number of trees built at each iteration. Equals 1 for regression and binary classification, or K where K is the number of classes for multiclass classification. + sample_weight : array-like of shape(n_samples,) default=None + Weights of training data. + Returns ------- gradients : ndarray, shape (prediction_dim, n_samples) @@ -63,6 +82,7 @@ def init_gradients_and_hessians(self, n_samples, prediction_dim): """ shape = (prediction_dim, n_samples) gradients = np.empty(shape=shape, dtype=G_H_DTYPE) + if self.hessians_are_constant: # If the hessians are constant, we consider they are equal to 1. # - This is correct for the half LS loss @@ -75,13 +95,17 @@ def init_gradients_and_hessians(self, n_samples, prediction_dim): return gradients, hessians @abstractmethod - def get_baseline_prediction(self, y_train, prediction_dim): + def get_baseline_prediction(self, y_train, sample_weight, prediction_dim): """Return initial predictions (before the first iteration). Parameters ---------- y_train : ndarray, shape (n_samples,) The target training values. + + sample_weight : array-like of shape(n_samples,) default=None + Weights of training data. + prediction_dim : int The dimension of one prediction: 1 for binary classification and regression, n_classes for multiclass classification. @@ -94,7 +118,7 @@ def get_baseline_prediction(self, y_train, prediction_dim): @abstractmethod def update_gradients_and_hessians(self, gradients, hessians, y_true, - raw_predictions): + raw_predictions, sample_weight): """Update gradients and hessians arrays, inplace. The gradients (resp. hessians) are the first (resp. second) order @@ -105,14 +129,20 @@ def update_gradients_and_hessians(self, gradients, hessians, y_true, ---------- gradients : ndarray, shape (prediction_dim, n_samples) The gradients (treated as OUT array). + hessians : ndarray, shape (prediction_dim, n_samples) or \ (1,) The hessians (treated as OUT array). + y_true : ndarray, shape (n_samples,) The true target values or each training sample. + raw_predictions : ndarray, shape (prediction_dim, n_samples) The raw_predictions (i.e. values from the trees) of the tree ensemble at iteration ``i - 1``. + + sample_weight : array-like of shape(n_samples,) default=None + Weights of training data. """ @@ -123,45 +153,58 @@ class LeastSquares(BaseLoss): loss(x_i) = 0.5 * (y_true_i - raw_pred_i)**2 - This actually computes the half least squares loss to optimize simplify + This actually computes the half least squares loss to simplify the computation of the gradients and get a unit hessian (and be consistent with what is done in LightGBM). """ + def __init__(self, sample_weight): + # If sample weights are provided, the hessians and gradients + # are multiplied by sample_weight, which means the hessians are + # equal to sample weights. + super().__init__(hessians_are_constant=sample_weight is None) - hessians_are_constant = True - - def __call__(self, y_true, raw_predictions, average=True): + def pointwise_loss(self, y_true, raw_predictions): # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to # return a view. raw_predictions = raw_predictions.reshape(-1) loss = 0.5 * np.power(y_true - raw_predictions, 2) - return loss.mean() if average else loss + return loss - def get_baseline_prediction(self, y_train, prediction_dim): - return np.mean(y_train) + def get_baseline_prediction(self, y_train, sample_weight, prediction_dim): + return np.average(y_train, weights=sample_weight) @staticmethod def inverse_link_function(raw_predictions): return raw_predictions def update_gradients_and_hessians(self, gradients, hessians, y_true, - raw_predictions): + raw_predictions, sample_weight): # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to # return a view. raw_predictions = raw_predictions.reshape(-1) gradients = gradients.reshape(-1) - _update_gradients_least_squares(gradients, y_true, raw_predictions) + if sample_weight is None: + _update_gradients_least_squares(gradients, y_true, raw_predictions) + else: + hessians = hessians.reshape(-1) + _update_gradients_hessians_least_squares(gradients, hessians, + y_true, raw_predictions, + sample_weight) class LeastAbsoluteDeviation(BaseLoss): - """Least asbolute deviation, for regression. + """Least absolute deviation, for regression. For a given sample x_i, the loss is defined as:: loss(x_i) = |y_true_i - raw_pred_i| """ + def __init__(self, sample_weight): + # If sample weights are provided, the hessians and gradients + # are multiplied by sample_weight, which means the hessians are + # equal to sample weights. + super().__init__(hessians_are_constant=sample_weight is None) - hessians_are_constant = True # This variable indicates whether the loss requires the leaves values to # be updated once the tree has been trained. The trees are trained to # predict a Newton-Raphson step (see grower._finalize_leaf()). But for @@ -172,30 +215,39 @@ class LeastAbsoluteDeviation(BaseLoss): # (https://statweb.stanford.edu/~jhf/ftp/trebst.pdf) for the theory. need_update_leaves_values = True - def __call__(self, y_true, raw_predictions, average=True): + def pointwise_loss(self, y_true, raw_predictions): # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to # return a view. raw_predictions = raw_predictions.reshape(-1) loss = np.abs(y_true - raw_predictions) - return loss.mean() if average else loss + return loss - def get_baseline_prediction(self, y_train, prediction_dim): - return np.median(y_train) + def get_baseline_prediction(self, y_train, sample_weight, prediction_dim): + if sample_weight is None: + return np.median(y_train) + else: + return _weighted_percentile(y_train, sample_weight, 50) @staticmethod def inverse_link_function(raw_predictions): return raw_predictions def update_gradients_and_hessians(self, gradients, hessians, y_true, - raw_predictions): + raw_predictions, sample_weight): # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to # return a view. raw_predictions = raw_predictions.reshape(-1) gradients = gradients.reshape(-1) - _update_gradients_least_absolute_deviation(gradients, y_true, - raw_predictions) + if sample_weight is None: + _update_gradients_least_absolute_deviation(gradients, y_true, + raw_predictions) + else: + hessians = hessians.reshape(-1) + _update_gradients_hessians_least_absolute_deviation( + gradients, hessians, y_true, raw_predictions, sample_weight) - def update_leaves_values(self, grower, y_true, raw_predictions): + def update_leaves_values(self, grower, y_true, raw_predictions, + sample_weight): # Update the values predicted by the tree with # median(y_true - raw_predictions). # See note about need_update_leaves_values in BaseLoss. @@ -205,7 +257,14 @@ def update_leaves_values(self, grower, y_true, raw_predictions): # requires a cython version of median() for leaf in grower.finalized_leaves: indices = leaf.sample_indices - median_res = np.median(y_true[indices] - raw_predictions[indices]) + if sample_weight is None: + median_res = np.median(y_true[indices] + - raw_predictions[indices]) + else: + median_res = _weighted_percentile(y_true[indices] + - raw_predictions[indices], + sample_weight=sample_weight, + percentile=50) leaf.value = grower.shrinkage * median_res # Note that the regularization is ignored here @@ -222,24 +281,26 @@ class BinaryCrossEntropy(BaseLoss): section 4.4.1 (about logistic regression). """ - hessians_are_constant = False + def __init__(self, sample_weight): + super().__init__(hessians_are_constant=False) + inverse_link_function = staticmethod(expit) - def __call__(self, y_true, raw_predictions, average=True): + def pointwise_loss(self, y_true, raw_predictions): # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to # return a view. raw_predictions = raw_predictions.reshape(-1) # logaddexp(0, x) = log(1 + exp(x)) loss = np.logaddexp(0, raw_predictions) - y_true * raw_predictions - return loss.mean() if average else loss + return loss - def get_baseline_prediction(self, y_train, prediction_dim): + def get_baseline_prediction(self, y_train, sample_weight, prediction_dim): if prediction_dim > 2: raise ValueError( "loss='binary_crossentropy' is not defined for multiclass" " classification with n_classes=%d, use" " loss='categorical_crossentropy' instead" % prediction_dim) - proba_positive_class = np.mean(y_train) + proba_positive_class = np.average(y_train, weights=sample_weight) eps = np.finfo(y_train.dtype).eps proba_positive_class = np.clip(proba_positive_class, eps, 1 - eps) # log(x / 1 - x) is the anti function of sigmoid, or the link function @@ -247,14 +308,14 @@ def get_baseline_prediction(self, y_train, prediction_dim): return np.log(proba_positive_class / (1 - proba_positive_class)) def update_gradients_and_hessians(self, gradients, hessians, y_true, - raw_predictions): + raw_predictions, sample_weight): # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to # return a view. raw_predictions = raw_predictions.reshape(-1) gradients = gradients.reshape(-1) hessians = hessians.reshape(-1) _update_gradients_hessians_binary_crossentropy( - gradients, hessians, y_true, raw_predictions) + gradients, hessians, y_true, raw_predictions, sample_weight) def predict_proba(self, raw_predictions): # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to @@ -274,9 +335,10 @@ class CategoricalCrossEntropy(BaseLoss): cross-entropy to more than 2 classes. """ - hessians_are_constant = False + def __init__(self, sample_weight): + super().__init__(hessians_are_constant=False) - def __call__(self, y_true, raw_predictions, average=True): + def pointwise_loss(self, y_true, raw_predictions): one_hot_true = np.zeros_like(raw_predictions) prediction_dim = raw_predictions.shape[0] for k in range(prediction_dim): @@ -284,22 +346,23 @@ def __call__(self, y_true, raw_predictions, average=True): loss = (logsumexp(raw_predictions, axis=0) - (one_hot_true * raw_predictions).sum(axis=0)) - return loss.mean() if average else loss + return loss - def get_baseline_prediction(self, y_train, prediction_dim): + def get_baseline_prediction(self, y_train, sample_weight, prediction_dim): init_value = np.zeros(shape=(prediction_dim, 1), dtype=Y_DTYPE) eps = np.finfo(y_train.dtype).eps for k in range(prediction_dim): - proba_kth_class = np.mean(y_train == k) + proba_kth_class = np.average(y_train == k, + weights=sample_weight) proba_kth_class = np.clip(proba_kth_class, eps, 1 - eps) init_value[k, :] += np.log(proba_kth_class) return init_value def update_gradients_and_hessians(self, gradients, hessians, y_true, - raw_predictions): + raw_predictions, sample_weight): _update_gradients_hessians_categorical_crossentropy( - gradients, hessians, y_true, raw_predictions) + gradients, hessians, y_true, raw_predictions, sample_weight) def predict_proba(self, raw_predictions): # TODO: This could be done in parallel diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_compare_lightgbm.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_compare_lightgbm.py index 32bb5dee4b197..61714d2dda775 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_compare_lightgbm.py +++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_compare_lightgbm.py @@ -13,9 +13,6 @@ get_equivalent_estimator) -pytest.importorskip("lightgbm") - - @pytest.mark.parametrize('seed', range(5)) @pytest.mark.parametrize('min_samples_leaf', (1, 20)) @pytest.mark.parametrize('n_samples, max_leaf_nodes', [ @@ -46,6 +43,7 @@ def test_same_predictions_regression(seed, min_samples_leaf, n_samples, # discrepancy between the initial values leads to biggish differences in # the predictions. These differences are much smaller with more # iterations. + pytest.importorskip("lightgbm") rng = np.random.RandomState(seed=seed) n_samples = n_samples @@ -66,7 +64,7 @@ def test_same_predictions_regression(seed, min_samples_leaf, n_samples, max_iter=max_iter, max_bins=max_bins, learning_rate=1, - n_iter_no_change=None, + early_stopping=False, min_samples_leaf=min_samples_leaf, max_leaf_nodes=max_leaf_nodes) est_lightgbm = get_equivalent_estimator(est_sklearn, lib='lightgbm') @@ -98,6 +96,7 @@ def test_same_predictions_regression(seed, min_samples_leaf, n_samples, def test_same_predictions_classification(seed, min_samples_leaf, n_samples, max_leaf_nodes): # Same as test_same_predictions_regression but for classification + pytest.importorskip("lightgbm") rng = np.random.RandomState(seed=seed) n_samples = n_samples @@ -119,7 +118,7 @@ def test_same_predictions_classification(seed, min_samples_leaf, n_samples, max_iter=max_iter, max_bins=max_bins, learning_rate=1, - n_iter_no_change=None, + early_stopping=False, min_samples_leaf=min_samples_leaf, max_leaf_nodes=max_leaf_nodes) est_lightgbm = get_equivalent_estimator(est_sklearn, lib='lightgbm') @@ -158,6 +157,7 @@ def test_same_predictions_classification(seed, min_samples_leaf, n_samples, def test_same_predictions_multiclass_classification( seed, min_samples_leaf, n_samples, max_leaf_nodes): # Same as test_same_predictions_regression but for classification + pytest.importorskip("lightgbm") rng = np.random.RandomState(seed=seed) n_samples = n_samples @@ -181,7 +181,7 @@ def test_same_predictions_multiclass_classification( max_iter=max_iter, max_bins=max_bins, learning_rate=lr, - n_iter_no_change=None, + early_stopping=False, min_samples_leaf=min_samples_leaf, max_leaf_nodes=max_leaf_nodes) est_lightgbm = get_equivalent_estimator(est_sklearn, lib='lightgbm') diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_gradient_boosting.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_gradient_boosting.py index 117539a424119..88ac63f7d05c9 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_gradient_boosting.py +++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_gradient_boosting.py @@ -1,6 +1,6 @@ import numpy as np import pytest -from numpy.testing import assert_allclose +from numpy.testing import assert_allclose, assert_array_equal from sklearn.datasets import make_classification, make_regression from sklearn.preprocessing import KBinsDiscretizer, MinMaxScaler from sklearn.model_selection import train_test_split @@ -11,6 +11,8 @@ from sklearn.experimental import enable_hist_gradient_boosting # noqa from sklearn.ensemble import HistGradientBoostingRegressor from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.ensemble._hist_gradient_boosting.loss import _LOSSES +from sklearn.ensemble._hist_gradient_boosting.grower import TreeGrower from sklearn.ensemble._hist_gradient_boosting.binning import _BinMapper from sklearn.utils import shuffle @@ -19,6 +21,14 @@ X_regression, y_regression = make_regression(random_state=0) +def _make_dumb_dataset(n_samples): + """Make a dumb dataset to test early stopping.""" + rng = np.random.RandomState(42) + X_dumb = rng.randn(n_samples, 1) + y_dumb = (X_dumb[:, 0] > 0).astype('int64') + return X_dumb, y_dumb + + @pytest.mark.parametrize('GradientBoosting, X, y', [ (HistGradientBoostingClassifier, X_classification, y_classification), (HistGradientBoostingRegressor, X_regression, y_regression) @@ -31,8 +41,7 @@ ({'max_iter': 0}, 'max_iter=0 must not be smaller than 1'), ({'max_leaf_nodes': 0}, 'max_leaf_nodes=0 should not be smaller than 2'), ({'max_leaf_nodes': 1}, 'max_leaf_nodes=1 should not be smaller than 2'), - ({'max_depth': 0}, 'max_depth=0 should not be smaller than 2'), - ({'max_depth': 1}, 'max_depth=1 should not be smaller than 2'), + ({'max_depth': 0}, 'max_depth=0 should not be smaller than 1'), ({'min_samples_leaf': 0}, 'min_samples_leaf=0 should not be smaller'), ({'l2_regularization': -1}, 'l2_regularization=-1 must be positive'), ({'max_bins': 1}, 'max_bins=1 should be no smaller than 2 and no larger'), @@ -58,17 +67,17 @@ def test_invalid_classification_loss(): @pytest.mark.parametrize( - 'scoring, validation_fraction, n_iter_no_change, tol', [ - ('neg_mean_squared_error', .1, 5, 1e-7), # use scorer - ('neg_mean_squared_error', None, 5, 1e-1), # use scorer on train data - (None, .1, 5, 1e-7), # same with default scorer - (None, None, 5, 1e-1), - ('loss', .1, 5, 1e-7), # use loss - ('loss', None, 5, 1e-1), # use loss on training data - (None, None, None, None), # no early stopping + 'scoring, validation_fraction, early_stopping, n_iter_no_change, tol', [ + ('neg_mean_squared_error', .1, True, 5, 1e-7), # use scorer + ('neg_mean_squared_error', None, True, 5, 1e-1), # use scorer on train + (None, .1, True, 5, 1e-7), # same with default scorer + (None, None, True, 5, 1e-1), + ('loss', .1, True, 5, 1e-7), # use loss + ('loss', None, True, 5, 1e-1), # use loss on training data + (None, None, False, 5, None), # no early stopping ]) def test_early_stopping_regression(scoring, validation_fraction, - n_iter_no_change, tol): + early_stopping, n_iter_no_change, tol): max_iter = 200 @@ -79,6 +88,7 @@ def test_early_stopping_regression(scoring, validation_fraction, min_samples_leaf=5, # easier to overfit fast scoring=scoring, tol=tol, + early_stopping=early_stopping, validation_fraction=validation_fraction, max_iter=max_iter, n_iter_no_change=n_iter_no_change, @@ -86,7 +96,7 @@ def test_early_stopping_regression(scoring, validation_fraction, ) gb.fit(X, y) - if n_iter_no_change is not None: + if early_stopping: assert n_iter_no_change <= gb.n_iter_ < max_iter else: assert gb.n_iter_ == max_iter @@ -98,17 +108,17 @@ def test_early_stopping_regression(scoring, validation_fraction, random_state=0) )) @pytest.mark.parametrize( - 'scoring, validation_fraction, n_iter_no_change, tol', [ - ('accuracy', .1, 5, 1e-7), # use scorer - ('accuracy', None, 5, 1e-1), # use scorer on training data - (None, .1, 5, 1e-7), # same with default scorerscor - (None, None, 5, 1e-1), - ('loss', .1, 5, 1e-7), # use loss - ('loss', None, 5, 1e-1), # use loss on training data - (None, None, None, None), # no early stopping + 'scoring, validation_fraction, early_stopping, n_iter_no_change, tol', [ + ('accuracy', .1, True, 5, 1e-7), # use scorer + ('accuracy', None, True, 5, 1e-1), # use scorer on training data + (None, .1, True, 5, 1e-7), # same with default scorer + (None, None, True, 5, 1e-1), + ('loss', .1, True, 5, 1e-7), # use loss + ('loss', None, True, 5, 1e-1), # use loss on training data + (None, None, False, 5, None), # no early stopping ]) def test_early_stopping_classification(data, scoring, validation_fraction, - n_iter_no_change, tol): + early_stopping, n_iter_no_change, tol): max_iter = 50 @@ -119,6 +129,7 @@ def test_early_stopping_classification(data, scoring, validation_fraction, min_samples_leaf=5, # easier to overfit fast scoring=scoring, tol=tol, + early_stopping=early_stopping, validation_fraction=validation_fraction, max_iter=max_iter, n_iter_no_change=n_iter_no_change, @@ -126,12 +137,29 @@ def test_early_stopping_classification(data, scoring, validation_fraction, ) gb.fit(X, y) - if n_iter_no_change is not None: + if early_stopping is True: assert n_iter_no_change <= gb.n_iter_ < max_iter else: assert gb.n_iter_ == max_iter +@pytest.mark.parametrize('GradientBoosting, X, y', [ + (HistGradientBoostingClassifier, *_make_dumb_dataset(10000)), + (HistGradientBoostingClassifier, *_make_dumb_dataset(10001)), + (HistGradientBoostingRegressor, *_make_dumb_dataset(10000)), + (HistGradientBoostingRegressor, *_make_dumb_dataset(10001)) +]) +def test_early_stopping_default(GradientBoosting, X, y): + # Test that early stopping is enabled by default if and only if there + # are more than 10000 samples + gb = GradientBoosting(max_iter=10, n_iter_no_change=2, tol=1e-1) + gb.fit(X, y) + if X.shape[0] > 10000: + assert gb.n_iter_ < gb.max_iter + else: + assert gb.n_iter_ == gb.max_iter + + @pytest.mark.parametrize( 'scores, n_iter_no_change, tol, stopping', [ @@ -171,7 +199,7 @@ def test_binning_train_validation_are_separated(): rng = np.random.RandomState(0) validation_fraction = .2 gb = HistGradientBoostingClassifier( - n_iter_no_change=5, + early_stopping=True, validation_fraction=validation_fraction, random_state=rng ) @@ -275,7 +303,8 @@ def test_small_trainset(): gb = HistGradientBoostingClassifier() # Compute the small training set - X_small, y_small = gb._get_small_trainset(X, y, seed=42) + X_small, y_small, _ = gb._get_small_trainset(X, y, seed=42, + sample_weight_train=None) # Compute the class distribution in the small training set unique, counts = np.unique(y_small, return_counts=True) @@ -409,11 +438,25 @@ def test_infinite_values(): np.testing.assert_allclose(gbdt.predict(X), y, atol=1e-4) +def test_consistent_lengths(): + X = np.array([-np.inf, 0, 1, np.inf]).reshape(-1, 1) + y = np.array([0, 0, 1, 1]) + sample_weight = np.array([.1, .3, .1]) + gbdt = HistGradientBoostingRegressor() + with pytest.raises(ValueError, + match=r"sample_weight.shape == \(3,\), expected"): + gbdt.fit(X, y, sample_weight) + + with pytest.raises(ValueError, + match="Found input variables with inconsistent number"): + gbdt.fit(X, y[1:]) + + def test_infinite_values_missing_values(): # High level test making sure that inf and nan values are properly handled # when both are present. This is similar to # test_split_on_nan_with_infinite_values() in test_grower.py, though we - # cannot check the predicitons for binned values here. + # cannot check the predictions for binned values here. X = np.asarray([-np.inf, 0, 1, np.inf, np.nan]).reshape(-1, 1) y_isnan = np.isnan(X.ravel()) @@ -446,3 +489,172 @@ def test_string_target_early_stopping(scoring): y = np.array(['x'] * 50 + ['y'] * 50, dtype=object) gbrt = HistGradientBoostingClassifier(n_iter_no_change=10, scoring=scoring) gbrt.fit(X, y) + + +def test_zero_sample_weights_regression(): + # Make sure setting a SW to zero amounts to ignoring the corresponding + # sample + + X = [[1, 0], + [1, 0], + [1, 0], + [0, 1]] + y = [0, 0, 1, 0] + # ignore the first 2 training samples by setting their weight to 0 + sample_weight = [0, 0, 1, 1] + gb = HistGradientBoostingRegressor(min_samples_leaf=1) + gb.fit(X, y, sample_weight=sample_weight) + assert gb.predict([[1, 0]])[0] > 0.5 + + +def test_zero_sample_weights_classification(): + # Make sure setting a SW to zero amounts to ignoring the corresponding + # sample + + X = [[1, 0], + [1, 0], + [1, 0], + [0, 1]] + y = [0, 0, 1, 0] + # ignore the first 2 training samples by setting their weight to 0 + sample_weight = [0, 0, 1, 1] + gb = HistGradientBoostingClassifier(loss='binary_crossentropy', + min_samples_leaf=1) + gb.fit(X, y, sample_weight=sample_weight) + assert_array_equal(gb.predict([[1, 0]]), [1]) + + X = [[1, 0], + [1, 0], + [1, 0], + [0, 1], + [1, 1]] + y = [0, 0, 1, 0, 2] + # ignore the first 2 training samples by setting their weight to 0 + sample_weight = [0, 0, 1, 1, 1] + gb = HistGradientBoostingClassifier(loss='categorical_crossentropy', + min_samples_leaf=1) + gb.fit(X, y, sample_weight=sample_weight) + assert_array_equal(gb.predict([[1, 0]]), [1]) + + +@pytest.mark.parametrize('problem', ( + 'regression', + 'binary_classification', + 'multiclass_classification' +)) +@pytest.mark.parametrize('duplication', ('half', 'all')) +def test_sample_weight_effect(problem, duplication): + # High level test to make sure that duplicating a sample is equivalent to + # giving it weight of 2. + + # fails for n_samples > 255 because binning does not take sample weights + # into account. Keeping n_samples <= 255 makes + # sure only unique values are used so SW have no effect on binning. + n_samples = 255 + n_features = 2 + if problem == 'regression': + X, y = make_regression(n_samples=n_samples, n_features=n_features, + n_informative=n_features, random_state=0) + Klass = HistGradientBoostingRegressor + else: + n_classes = 2 if problem == 'binary_classification' else 3 + X, y = make_classification(n_samples=n_samples, n_features=n_features, + n_informative=n_features, n_redundant=0, + n_clusters_per_class=1, + n_classes=n_classes, random_state=0) + Klass = HistGradientBoostingClassifier + + # This test can't pass if min_samples_leaf > 1 because that would force 2 + # samples to be in the same node in est_sw, while these samples would be + # free to be separate in est_dup: est_dup would just group together the + # duplicated samples. + est = Klass(min_samples_leaf=1) + + # Create dataset with duplicate and corresponding sample weights + if duplication == 'half': + lim = n_samples // 2 + else: + lim = n_samples + X_dup = np.r_[X, X[:lim]] + y_dup = np.r_[y, y[:lim]] + sample_weight = np.ones(shape=(n_samples)) + sample_weight[:lim] = 2 + + est_sw = clone(est).fit(X, y, sample_weight=sample_weight) + est_dup = clone(est).fit(X_dup, y_dup) + + # checking raw_predict is stricter than just predict for classification + assert np.allclose(est_sw._raw_predict(X_dup), + est_dup._raw_predict(X_dup)) + + +@pytest.mark.parametrize('loss_name', ('least_squares', + 'least_absolute_deviation')) +def test_sum_hessians_are_sample_weight(loss_name): + # For losses with constant hessians, the sum_hessians field of the + # histograms must be equal to the sum of the sample weight of samples at + # the corresponding bin. + + rng = np.random.RandomState(0) + n_samples = 1000 + n_features = 2 + X, y = make_regression(n_samples=n_samples, n_features=n_features, + random_state=rng) + bin_mapper = _BinMapper() + X_binned = bin_mapper.fit_transform(X) + + sample_weight = rng.normal(size=n_samples) + + loss = _LOSSES[loss_name](sample_weight=sample_weight) + gradients, hessians = loss.init_gradients_and_hessians( + n_samples=n_samples, prediction_dim=1, sample_weight=sample_weight) + raw_predictions = rng.normal(size=(1, n_samples)) + loss.update_gradients_and_hessians(gradients, hessians, y, + raw_predictions, sample_weight) + + # build sum_sample_weight which contains the sum of the sample weights at + # each bin (for each feature). This must be equal to the sum_hessians + # field of the corresponding histogram + sum_sw = np.zeros(shape=(n_features, bin_mapper.n_bins)) + for feature_idx in range(n_features): + for sample_idx in range(n_samples): + sum_sw[feature_idx, X_binned[sample_idx, feature_idx]] += ( + sample_weight[sample_idx]) + + # Build histogram + grower = TreeGrower(X_binned, gradients[0], hessians[0], + n_bins=bin_mapper.n_bins) + histograms = grower.histogram_builder.compute_histograms_brute( + grower.root.sample_indices) + + for feature_idx in range(n_features): + for bin_idx in range(bin_mapper.n_bins): + assert histograms[feature_idx, bin_idx]['sum_hessians'] == ( + pytest.approx(sum_sw[feature_idx, bin_idx], rel=1e-5)) + + +def test_max_depth_max_leaf_nodes(): + # Non regression test for + # https://github.com/scikit-learn/scikit-learn/issues/16179 + # there was a bug when the max_depth and the max_leaf_nodes criteria were + # met at the same time, which would lead to max_leaf_nodes not being + # respected. + X, y = make_classification(random_state=0) + est = HistGradientBoostingClassifier(max_depth=2, max_leaf_nodes=3, + max_iter=1).fit(X, y) + tree = est._predictors[0][0] + assert tree.get_max_depth() == 2 + assert tree.get_n_leaf_nodes() == 3 # would be 4 prior to bug fix + + +def test_early_stopping_on_test_set_with_warm_start(): + # Non regression test for #16661 where second fit fails with + # warm_start=True, early_stopping is on, and no validation set + X, y = make_classification(random_state=0) + gb = HistGradientBoostingClassifier( + max_iter=1, scoring='loss', warm_start=True, early_stopping=True, + n_iter_no_change=1, validation_fraction=None) + + gb.fit(X, y) + # does not raise on second call + gb.fit(X, y) diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_grower.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_grower.py index 0cc301b7b1b36..d770b50e7aa30 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_grower.py +++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_grower.py @@ -257,7 +257,14 @@ def test_min_samples_leaf_root(n_samples, min_samples_leaf): assert len(grower.finalized_leaves) == 1 -@pytest.mark.parametrize('max_depth', [2, 3]) +def assert_is_stump(grower): + # To assert that stumps are created when max_depth=1 + for leaf in (grower.root.left_child, grower.root.right_child): + assert leaf.left_child is None + assert leaf.right_child is None + + +@pytest.mark.parametrize('max_depth', [1, 2, 3]) def test_max_depth(max_depth): # Make sure max_depth parameter works as expected rng = np.random.RandomState(seed=0) @@ -279,6 +286,9 @@ def test_max_depth(max_depth): depth = max(leaf.depth for leaf in grower.finalized_leaves) assert depth == max_depth + if max_depth == 1: + assert_is_stump(grower) + def test_input_validation(): diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_loss.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_loss.py index 8c300db993d3d..915dc300e4760 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_loss.py +++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_loss.py @@ -20,7 +20,7 @@ def get_gradients(y_true, raw_predictions): gradients = np.empty_like(raw_predictions, dtype=G_H_DTYPE) hessians = np.empty_like(raw_predictions, dtype=G_H_DTYPE) loss.update_gradients_and_hessians(gradients, hessians, y_true, - raw_predictions) + raw_predictions, None) return gradients def get_hessians(y_true, raw_predictions): @@ -28,7 +28,7 @@ def get_hessians(y_true, raw_predictions): gradients = np.empty_like(raw_predictions, dtype=G_H_DTYPE) hessians = np.empty_like(raw_predictions, dtype=G_H_DTYPE) loss.update_gradients_and_hessians(gradients, hessians, y_true, - raw_predictions) + raw_predictions, None) if loss.__class__.__name__ == 'LeastSquares': # hessians aren't updated because they're constant: @@ -62,13 +62,13 @@ def test_derivatives(loss, x0, y_true): # using Halley's method with the first and second order derivatives # computed by the Loss instance. - loss = _LOSSES[loss]() + loss = _LOSSES[loss](sample_weight=None) y_true = np.array([y_true], dtype=Y_DTYPE) x0 = np.array([x0], dtype=Y_DTYPE).reshape(1, 1) get_gradients, get_hessians = get_derivatives_helper(loss) def func(x): - return loss(y_true, x) + return loss.pointwise_loss(y_true, x) def fprime(x): return get_gradients(y_true, x) @@ -78,7 +78,7 @@ def fprime2(x): optimum = newton(func, x0=x0, fprime=fprime, fprime2=fprime2) assert np.allclose(loss.inverse_link_function(optimum), y_true) - assert np.allclose(loss(y_true, optimum), 0) + assert np.allclose(loss.pointwise_loss(y_true, optimum), 0) assert np.allclose(get_gradients(y_true, optimum), 0) @@ -105,7 +105,7 @@ def test_numerical_gradients(loss, n_classes, prediction_dim, seed=0): raw_predictions = rng.normal( size=(prediction_dim, n_samples) ).astype(Y_DTYPE) - loss = _LOSSES[loss]() + loss = _LOSSES[loss](sample_weight=None) get_gradients, get_hessians = get_derivatives_helper(loss) # only take gradients and hessians of first tree / class. @@ -120,16 +120,16 @@ def test_numerical_gradients(loss, n_classes, prediction_dim, seed=0): eps = 1e-9 offset = np.zeros_like(raw_predictions) offset[0, :] = eps - f_plus_eps = loss(y_true, raw_predictions + offset / 2, average=False) - f_minus_eps = loss(y_true, raw_predictions - offset / 2, average=False) + f_plus_eps = loss.pointwise_loss(y_true, raw_predictions + offset / 2) + f_minus_eps = loss.pointwise_loss(y_true, raw_predictions - offset / 2) numerical_gradients = (f_plus_eps - f_minus_eps) / eps # Approximate hessians eps = 1e-4 # need big enough eps as we divide by its square offset[0, :] = eps - f_plus_eps = loss(y_true, raw_predictions + offset, average=False) - f_minus_eps = loss(y_true, raw_predictions - offset, average=False) - f = loss(y_true, raw_predictions, average=False) + f_plus_eps = loss.pointwise_loss(y_true, raw_predictions + offset) + f_minus_eps = loss.pointwise_loss(y_true, raw_predictions - offset) + f = loss.pointwise_loss(y_true, raw_predictions) numerical_hessians = (f_plus_eps + f_minus_eps - 2 * f) / eps**2 assert_allclose(numerical_gradients, gradients, rtol=1e-4, atol=1e-7) @@ -139,9 +139,9 @@ def test_numerical_gradients(loss, n_classes, prediction_dim, seed=0): def test_baseline_least_squares(): rng = np.random.RandomState(0) - loss = _LOSSES['least_squares']() + loss = _LOSSES['least_squares'](sample_weight=None) y_train = rng.normal(size=100) - baseline_prediction = loss.get_baseline_prediction(y_train, 1) + baseline_prediction = loss.get_baseline_prediction(y_train, None, 1) assert baseline_prediction.shape == tuple() # scalar assert baseline_prediction.dtype == y_train.dtype # Make sure baseline prediction is the mean of all targets @@ -153,9 +153,9 @@ def test_baseline_least_squares(): def test_baseline_least_absolute_deviation(): rng = np.random.RandomState(0) - loss = _LOSSES['least_absolute_deviation']() + loss = _LOSSES['least_absolute_deviation'](sample_weight=None) y_train = rng.normal(size=100) - baseline_prediction = loss.get_baseline_prediction(y_train, 1) + baseline_prediction = loss.get_baseline_prediction(y_train, None, 1) assert baseline_prediction.shape == tuple() # scalar assert baseline_prediction.dtype == y_train.dtype # Make sure baseline prediction is the median of all targets @@ -167,10 +167,10 @@ def test_baseline_least_absolute_deviation(): def test_baseline_binary_crossentropy(): rng = np.random.RandomState(0) - loss = _LOSSES['binary_crossentropy']() + loss = _LOSSES['binary_crossentropy'](sample_weight=None) for y_train in (np.zeros(shape=100), np.ones(shape=100)): y_train = y_train.astype(np.float64) - baseline_prediction = loss.get_baseline_prediction(y_train, 1) + baseline_prediction = loss.get_baseline_prediction(y_train, None, 1) assert_all_finite(baseline_prediction) assert np.allclose(loss.inverse_link_function(baseline_prediction), y_train[0]) @@ -181,7 +181,7 @@ def test_baseline_binary_crossentropy(): # p = inverse_link_function(raw_prediction) = sigmoid(raw_prediction) # So we want raw_prediction = link_function(p) = log(p / (1 - p)) y_train = rng.randint(0, 2, size=100).astype(np.float64) - baseline_prediction = loss.get_baseline_prediction(y_train, 1) + baseline_prediction = loss.get_baseline_prediction(y_train, None, 1) assert baseline_prediction.shape == tuple() # scalar assert baseline_prediction.dtype == y_train.dtype p = y_train.mean() @@ -192,10 +192,10 @@ def test_baseline_categorical_crossentropy(): rng = np.random.RandomState(0) prediction_dim = 4 - loss = _LOSSES['categorical_crossentropy']() + loss = _LOSSES['categorical_crossentropy'](sample_weight=None) for y_train in (np.zeros(shape=100), np.ones(shape=100)): y_train = y_train.astype(np.float64) - baseline_prediction = loss.get_baseline_prediction(y_train, + baseline_prediction = loss.get_baseline_prediction(y_train, None, prediction_dim) assert baseline_prediction.dtype == y_train.dtype assert_all_finite(baseline_prediction) @@ -203,8 +203,85 @@ def test_baseline_categorical_crossentropy(): # Same logic as for above test. Here inverse_link_function = softmax and # link_function = log y_train = rng.randint(0, prediction_dim + 1, size=100).astype(np.float32) - baseline_prediction = loss.get_baseline_prediction(y_train, prediction_dim) + baseline_prediction = loss.get_baseline_prediction(y_train, None, + prediction_dim) assert baseline_prediction.shape == (prediction_dim, 1) for k in range(prediction_dim): p = (y_train == k).mean() assert np.allclose(baseline_prediction[k, :], np.log(p)) + + +@pytest.mark.parametrize('loss, problem', [ + ('least_squares', 'regression'), + ('least_absolute_deviation', 'regression'), + ('binary_crossentropy', 'classification'), + ('categorical_crossentropy', 'classification') + ]) +@pytest.mark.parametrize('sample_weight', ['ones', 'random']) +def test_sample_weight_multiplies_gradients(loss, problem, sample_weight): + # Make sure that passing sample weights to the gradient and hessians + # computation methods is equivalent to multiplying by the weights. + + rng = np.random.RandomState(42) + n_samples = 1000 + + if loss == 'categorical_crossentropy': + n_classes = prediction_dim = 3 + else: + n_classes = prediction_dim = 1 + + if problem == 'regression': + y_true = rng.normal(size=n_samples).astype(Y_DTYPE) + else: + y_true = rng.randint(0, n_classes, size=n_samples).astype(Y_DTYPE) + + if sample_weight == 'ones': + sample_weight = np.ones(shape=n_samples, dtype=Y_DTYPE) + else: + sample_weight = rng.normal(size=n_samples).astype(Y_DTYPE) + + loss_ = _LOSSES[loss](sample_weight=sample_weight) + + baseline_prediction = loss_.get_baseline_prediction( + y_true, None, prediction_dim + ) + raw_predictions = np.zeros(shape=(prediction_dim, n_samples), + dtype=baseline_prediction.dtype) + raw_predictions += baseline_prediction + + gradients = np.empty(shape=(prediction_dim, n_samples), dtype=G_H_DTYPE) + hessians = np.ones(shape=(prediction_dim, n_samples), dtype=G_H_DTYPE) + loss_.update_gradients_and_hessians(gradients, hessians, y_true, + raw_predictions, None) + + gradients_sw = np.empty(shape=(prediction_dim, n_samples), dtype=G_H_DTYPE) + hessians_sw = np.ones(shape=(prediction_dim, n_samples), dtype=G_H_DTYPE) + loss_.update_gradients_and_hessians(gradients_sw, hessians_sw, y_true, + raw_predictions, sample_weight) + + assert np.allclose(gradients * sample_weight, gradients_sw) + assert np.allclose(hessians * sample_weight, hessians_sw) + + +def test_init_gradient_and_hessians_sample_weight(): + # Make sure that passing sample_weight to a loss correctly influences the + # hessians_are_constant attribute, and consequently the shape of the + # hessians array. + + prediction_dim = 2 + n_samples = 5 + sample_weight = None + loss = _LOSSES['least_squares'](sample_weight=sample_weight) + _, hessians = loss.init_gradients_and_hessians( + n_samples=n_samples, prediction_dim=prediction_dim, + sample_weight=None) + assert loss.hessians_are_constant + assert hessians.shape == (1, 1) + + sample_weight = np.ones(n_samples) + loss = _LOSSES['least_squares'](sample_weight=sample_weight) + _, hessians = loss.init_gradients_and_hessians( + n_samples=n_samples, prediction_dim=prediction_dim, + sample_weight=sample_weight) + assert not loss.hessians_are_constant + assert hessians.shape == (prediction_dim, n_samples) diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_warm_start.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_warm_start.py index 5fcae12873a43..2417de4f6cc63 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_warm_start.py +++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_warm_start.py @@ -11,6 +11,7 @@ from sklearn.experimental import enable_hist_gradient_boosting # noqa from sklearn.ensemble import HistGradientBoostingRegressor from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.metrics import check_scoring X_classification, y_classification = make_classification(random_state=0) @@ -37,10 +38,11 @@ def test_max_iter_with_warm_start_validation(GradientBoosting, X, y): # is smaller than the number of iterations from the previous fit when warm # start is True. - estimator = GradientBoosting(max_iter=50, warm_start=True) + estimator = GradientBoosting(max_iter=10, early_stopping=False, + warm_start=True) estimator.fit(X, y) - estimator.set_params(max_iter=25) - err_msg = ('max_iter=25 must be larger than or equal to n_iter_=50 ' + estimator.set_params(max_iter=5) + err_msg = ('max_iter=5 must be larger than or equal to n_iter_=10 ' 'when warm_start==True') with pytest.raises(ValueError, match=err_msg): estimator.fit(X, y) @@ -75,14 +77,14 @@ def test_warm_start_yields_identical_results(GradientBoosting, X, y): ]) def test_warm_start_max_depth(GradientBoosting, X, y): # Test if possible to fit trees of different depth in ensemble. - gb = GradientBoosting(max_iter=100, min_samples_leaf=1, - warm_start=True, max_depth=2) + gb = GradientBoosting(max_iter=20, min_samples_leaf=1, + warm_start=True, max_depth=2, early_stopping=False) gb.fit(X, y) - gb.set_params(max_iter=110, max_depth=3) + gb.set_params(max_iter=30, max_depth=3, n_iter_no_change=110) gb.fit(X, y) - # First 100 trees have max_depth == 2 - for i in range(100): + # First 20 trees have max_depth == 2 + for i in range(20): assert gb._predictors[i][0].get_max_depth() == 2 # Last 10 trees have max_depth == 3 for i in range(1, 11): @@ -93,20 +95,21 @@ def test_warm_start_max_depth(GradientBoosting, X, y): (HistGradientBoostingClassifier, X_classification, y_classification), (HistGradientBoostingRegressor, X_regression, y_regression) ]) -def test_warm_start_early_stopping(GradientBoosting, X, y): +@pytest.mark.parametrize('scoring', (None, 'loss')) +def test_warm_start_early_stopping(GradientBoosting, X, y, scoring): # Make sure that early stopping occurs after a small number of iterations # when fitting a second time with warm starting. n_iter_no_change = 5 gb = GradientBoosting( - n_iter_no_change=n_iter_no_change, max_iter=10000, - random_state=42, warm_start=True, tol=1e-3 + n_iter_no_change=n_iter_no_change, max_iter=10000, early_stopping=True, + random_state=42, warm_start=True, tol=1e-3, scoring=scoring, ) gb.fit(X, y) n_iter_first_fit = gb.n_iter_ gb.fit(X, y) n_iter_second_fit = gb.n_iter_ - assert n_iter_second_fit - n_iter_first_fit < n_iter_no_change + assert 0 < n_iter_second_fit - n_iter_first_fit < n_iter_no_change @pytest.mark.parametrize('GradientBoosting, X, y', [ @@ -115,11 +118,12 @@ def test_warm_start_early_stopping(GradientBoosting, X, y): ]) def test_warm_start_equal_n_estimators(GradientBoosting, X, y): # Test if warm start with equal n_estimators does nothing - gb_1 = GradientBoosting(max_depth=2) + gb_1 = GradientBoosting(max_depth=2, early_stopping=False) gb_1.fit(X, y) gb_2 = clone(gb_1) - gb_2.set_params(max_iter=gb_1.max_iter, warm_start=True) + gb_2.set_params(max_iter=gb_1.max_iter, warm_start=True, + n_iter_no_change=5) gb_2.fit(X, y) # Check that both predictors are equal @@ -168,8 +172,9 @@ def _get_rng(rng_type): return np.random.RandomState(0) random_state = _get_rng(rng_type) - gb_1 = GradientBoosting(n_iter_no_change=5, max_iter=2, + gb_1 = GradientBoosting(early_stopping=True, max_iter=2, random_state=random_state) + gb_1.set_params(scoring=check_scoring(gb_1)) gb_1.fit(X, y) random_seed_1_1 = gb_1._random_seed @@ -177,8 +182,9 @@ def _get_rng(rng_type): random_seed_1_2 = gb_1._random_seed # clear the old state, different seed random_state = _get_rng(rng_type) - gb_2 = GradientBoosting(n_iter_no_change=5, max_iter=2, + gb_2 = GradientBoosting(early_stopping=True, max_iter=2, random_state=random_state, warm_start=True) + gb_2.set_params(scoring=check_scoring(gb_2)) gb_2.fit(X, y) # inits state random_seed_2_1 = gb_2._random_seed gb_2.fit(X, y) # clears old state and equals est diff --git a/sklearn/ensemble/_hist_gradient_boosting/utils.pyx b/sklearn/ensemble/_hist_gradient_boosting/utils.pyx index 4b1188b87e69e..cf2c5a51c90dd 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/utils.pyx +++ b/sklearn/ensemble/_hist_gradient_boosting/utils.pyx @@ -38,7 +38,7 @@ def get_equivalent_estimator(estimator, lib='lightgbm'): if sklearn_params['loss'] == 'auto': raise ValueError('auto loss is not accepted. We need to know if ' 'the problem is binary or multiclass classification.') - if sklearn_params['n_iter_no_change'] is not None: + if sklearn_params['early_stopping']: raise NotImplementedError('Early stopping should be deactivated.') lightgbm_loss_mapping = { diff --git a/sklearn/ensemble/_iforest.py b/sklearn/ensemble/_iforest.py index fcd9fe63ec755..501f2425541e8 100644 --- a/sklearn/ensemble/_iforest.py +++ b/sklearn/ensemble/_iforest.py @@ -50,10 +50,10 @@ class IsolationForest(OutlierMixin, BaseBagging): Parameters ---------- - n_estimators : int, optional (default=100) + n_estimators : int, default=100 The number of base estimators in the ensemble. - max_samples : int or float, optional (default="auto") + max_samples : "auto", int or float, default="auto" The number of samples to draw from X to train each base estimator. - If int, then draw `max_samples` samples. - If float, then draw `max_samples * X.shape[0]` samples. @@ -62,7 +62,7 @@ class IsolationForest(OutlierMixin, BaseBagging): If max_samples is larger than the number of samples provided, all samples will be used for all trees (no sampling). - contamination : 'auto' or float, optional (default='auto') + contamination : 'auto' or float, default='auto' The amount of contamination of the data set, i.e. the proportion of outliers in the data set. Used when fitting to define the threshold on the scores of the samples. @@ -75,18 +75,18 @@ class IsolationForest(OutlierMixin, BaseBagging): The default value of ``contamination`` changed from 0.1 to ``'auto'``. - max_features : int or float, optional (default=1.0) + max_features : int or float, default=1.0 The number of features to draw from X to train each base estimator. - If int, then draw `max_features` features. - If float, then draw `max_features * X.shape[1]` features. - bootstrap : bool, optional (default=False) + bootstrap : bool, default=False If True, individual trees are fit on random subsets of the training data sampled with replacement. If False, sampling without replacement is performed. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of jobs to run in parallel for both :meth:`fit` and :meth:`predict`. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all @@ -106,16 +106,17 @@ class IsolationForest(OutlierMixin, BaseBagging): ``behaviour`` parameter is deprecated in 0.22 and removed in 0.24. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState, default=None + Controls the pseudo-randomness of the selection of the feature + and split values for each branching step and each tree in the forest. - verbose : int, optional (default=0) + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary `. + + verbose : int, default=0 Controls the verbosity of the tree building process. - warm_start : bool, optional (default=False) + warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just fit a whole new forest. See :term:`the Glossary `. @@ -131,8 +132,8 @@ class IsolationForest(OutlierMixin, BaseBagging): The subset of drawn samples (i.e., the in-bag samples) for each base estimator. - max_samples_ : integer - The actual number of samples + max_samples_ : int + The actual number of samples. offset_ : float Offset used to define the decision function from the raw scores. We @@ -144,6 +145,9 @@ class IsolationForest(OutlierMixin, BaseBagging): is defined in such a way we obtain the expected number of outliers (samples with decision function < 0) in training. + estimators_features_ : list of arrays + The subset of drawn features for each base estimator. + Notes ----- The implementation is based on an ensemble of ExtraTreeRegressor. The @@ -224,7 +228,7 @@ def fit(self, X, y=None, sample_weight=None): Parameters ---------- - X : array-like or sparse matrix, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Use ``dtype=np.float32`` for maximum efficiency. Sparse matrices are also supported, use sparse ``csc_matrix`` for maximum efficiency. @@ -312,14 +316,14 @@ def predict(self, X): Parameters ---------- - X : array-like or sparse matrix, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. Returns ------- - is_inlier : array, shape (n_samples,) + is_inlier : ndarray of shape (n_samples,) For each observation, tells whether or not (+1 or -1) it should be considered as an inlier according to the fitted model. """ @@ -344,14 +348,14 @@ def decision_function(self, X): Parameters ---------- - X : array-like or sparse matrix, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. Returns ------- - scores : array, shape (n_samples,) + scores : ndarray of shape (n_samples,) The anomaly score of the input samples. The lower, the more abnormal. Negative scores represent outliers, positive scores represent inliers. @@ -376,12 +380,12 @@ def score_samples(self, X): Parameters ---------- - X : array-like or sparse matrix, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Returns ------- - scores : array, shape (n_samples,) + scores : ndarray of shape (n_samples,) The anomaly score of the input samples. The lower, the more abnormal. """ @@ -439,9 +443,10 @@ def _compute_score_samples(self, X, subsample_features): Parameters ---------- X : array-like or sparse matrix + Data matrix. - subsample_features : bool, - whether features should be subsampled + subsample_features : bool + Whether features should be subsampled. """ n_samples = X.shape[0] @@ -475,13 +480,13 @@ def _average_path_length(n_samples_leaf): latter has the same structure as an isolation tree. Parameters ---------- - n_samples_leaf : array-like, shape (n_samples,). + n_samples_leaf : array-like of shape (n_samples,) The number of training samples in each test sample leaf, for each estimators. Returns ------- - average_path_length : array, same shape as n_samples_leaf + average_path_length : ndarray of shape (n_samples,) """ n_samples_leaf = check_array(n_samples_leaf, ensure_2d=False) diff --git a/sklearn/ensemble/_stacking.py b/sklearn/ensemble/_stacking.py index c18291b9f4461..ba817613523f6 100644 --- a/sklearn/ensemble/_stacking.py +++ b/sklearn/ensemble/_stacking.py @@ -14,7 +14,7 @@ from ..base import ClassifierMixin, RegressorMixin, TransformerMixin from ..base import is_classifier, is_regressor -from ._base import _parallel_fit_estimator +from ._base import _fit_single_estimator from ._base import _BaseHeterogeneousEnsemble from ..linear_model import LogisticRegression @@ -63,7 +63,7 @@ def _concatenate_predictions(self, X, predictions): and `self.passthrough` is True, the output of `transform` will be sparse. - This helper is in charge of ensuring the preditions are 2D arrays and + This helper is in charge of ensuring the predictions are 2D arrays and it will drop one of the probability column when using probabilities in the binary case. Indeed, the p(y|c=0) = 1 - p(y|c=1) """ @@ -117,11 +117,15 @@ def fit(self, X, y, sample_weight=None): y : array-like of shape (n_samples,) Target values. - sample_weight : array-like of shape (n_samples,) or None + sample_weight : array-like of shape (n_samples,) or default=None Sample weights. If None, then samples are equally weighted. Note that this is supported only if all underlying estimators support sample weights. + .. versionchanged:: 0.23 + when not None, `sample_weight` is passed to all underlying + estimators + Returns ------- self : object @@ -137,9 +141,10 @@ def fit(self, X, y, sample_weight=None): # base estimators will be used in transform, predict, and # predict_proba. They are exposed publicly. self.estimators_ = Parallel(n_jobs=self.n_jobs)( - delayed(_parallel_fit_estimator)(clone(est), X, y, sample_weight) + delayed(_fit_single_estimator)(clone(est), X, y, sample_weight) for est in all_estimators if est != 'drop' ) + self.n_features_in_ = self.estimators_[0].n_features_in_ self.named_estimators_ = Bunch() est_fitted_idx = 0 @@ -165,10 +170,13 @@ def fit(self, X, y, sample_weight=None): self._method_name(name, est, meth) for name, est, meth in zip(names, all_estimators, stack_method) ] - + fit_params = ({"sample_weight": sample_weight} + if sample_weight is not None + else None) predictions = Parallel(n_jobs=self.n_jobs)( delayed(cross_val_predict)(clone(est), X, y, cv=deepcopy(cv), method=meth, n_jobs=self.n_jobs, + fit_params=fit_params, verbose=self.verbose) for est, meth in zip(all_estimators, self.stack_method_) if est != 'drop' @@ -182,21 +190,8 @@ def fit(self, X, y, sample_weight=None): ] X_meta = self._concatenate_predictions(X, predictions) - if sample_weight is not None: - try: - self.final_estimator_.fit( - X_meta, y, sample_weight=sample_weight - ) - except TypeError as exc: - if "unexpected keyword argument 'sample_weight'" in str(exc): - raise TypeError( - "Underlying estimator {} does not support sample " - "weights." - .format(self.final_estimator_.__class__.__name__) - ) from exc - raise - else: - self.final_estimator_.fit(X_meta, y) + _fit_single_estimator(self.final_estimator_, X_meta, y, + sample_weight=sample_weight) return self @@ -310,14 +305,20 @@ class StackingClassifier(ClassifierMixin, _BaseStacking): `final_estimator` is trained on the predictions as well as the original training data. + verbose : int, default=0 + Verbosity level. + Attributes ---------- + classes_ : ndarray of shape (n_classes,) + Class labels. + estimators_ : list of estimators The elements of the estimators parameter, having been fitted on the training data. If an estimator has been set to `'drop'`, it will not appear in `estimators_`. - named_estimators_ : Bunch + named_estimators_ : :class:`~sklearn.utils.Bunch` Attribute to access any fitted sub-estimators by name. final_estimator_ : estimator @@ -398,7 +399,7 @@ def fit(self, X, y, sample_weight=None): y : array-like of shape (n_samples,) Target values. - sample_weight : array-like of shape (n_samples,) or None + sample_weight : array-like of shape (n_samples,), default=None Sample weights. If None, then samples are equally weighted. Note that this is supported only if all underlying estimators support sample weights. @@ -555,6 +556,9 @@ class StackingRegressor(RegressorMixin, _BaseStacking): `final_estimator` is trained on the predictions as well as the original training data. + verbose : int, default=0 + Verbosity level. + Attributes ---------- estimators_ : list of estimator @@ -562,9 +566,10 @@ class StackingRegressor(RegressorMixin, _BaseStacking): training data. If an estimator has been set to `'drop'`, it will not appear in `estimators_`. - named_estimators_ : Bunch + named_estimators_ : :class:`~sklearn.utils.Bunch` Attribute to access any fitted sub-estimators by name. + final_estimator_ : estimator The regressor to stacked the base estimators fitted. @@ -630,7 +635,7 @@ def fit(self, X, y, sample_weight=None): y : array-like of shape (n_samples,) Target values. - sample_weight : array-like of shape (n_samples,) or None + sample_weight : array-like of shape (n_samples,), default=None Sample weights. If None, then samples are equally weighted. Note that this is supported only if all underlying estimators support sample weights. diff --git a/sklearn/ensemble/_voting.py b/sklearn/ensemble/_voting.py index bddf5f00b10af..0da6dc86c30fa 100644 --- a/sklearn/ensemble/_voting.py +++ b/sklearn/ensemble/_voting.py @@ -23,13 +23,14 @@ from ..base import RegressorMixin from ..base import TransformerMixin from ..base import clone -from ._base import _parallel_fit_estimator +from ._base import _fit_single_estimator from ._base import _BaseHeterogeneousEnsemble from ..preprocessing import LabelEncoder from ..utils import Bunch from ..utils.validation import check_is_fitted from ..utils.multiclass import check_classification_targets from ..utils.validation import column_or_1d +from ..exceptions import NotFittedError class _BaseVoting(TransformerMixin, _BaseHeterogeneousEnsemble): @@ -39,23 +40,26 @@ class _BaseVoting(TransformerMixin, _BaseHeterogeneousEnsemble): instead. """ + def _log_message(self, name, idx, total): + if not self.verbose: + return None + return '(%d of %d) Processing %s' % (idx, total, name) + @property def _weights_not_none(self): - """Get the weights of not `None` estimators""" + """Get the weights of not `None` estimators.""" if self.weights is None: return None return [w for est, w in zip(self.estimators, self.weights) if est[1] not in (None, 'drop')] def _predict(self, X): - """Collect results from clf.predict calls. """ + """Collect results from clf.predict calls.""" return np.asarray([est.predict(X) for est in self.estimators_]).T @abstractmethod def fit(self, X, y, sample_weight=None): - """ - common fit operations. - """ + """Get common fit operations.""" names, clfs = self._validate_estimators() if (self.weights is not None and @@ -65,9 +69,14 @@ def fit(self, X, y, sample_weight=None): % (len(self.weights), len(self.estimators))) self.estimators_ = Parallel(n_jobs=self.n_jobs)( - delayed(_parallel_fit_estimator)(clone(clf), X, y, - sample_weight=sample_weight) - for clf in clfs if clf not in (None, 'drop') + delayed(_fit_single_estimator)( + clone(clf), X, y, + sample_weight=sample_weight, + message_clsname='Voting', + message=self._log_message(names[idx], + idx + 1, len(clfs)) + ) + for idx, clf in enumerate(clfs) if clf not in (None, 'drop') ) self.named_estimators_ = Bunch() @@ -80,6 +89,20 @@ def fit(self, X, y, sample_weight=None): return self + @property + def n_features_in_(self): + # For consistency with other estimators we raise a AttributeError so + # that hasattr() fails if the estimator isn't fitted. + try: + check_is_fitted(self) + except NotFittedError as nfe: + raise AttributeError( + "{} object has no n_features_in_ attribute." + .format(self.__class__.__name__) + ) from nfe + + return self.estimators_[0].n_features_in_ + class VotingClassifier(ClassifierMixin, _BaseVoting): """Soft Voting/Majority Rule classifier for unfitted estimators. @@ -90,7 +113,7 @@ class VotingClassifier(ClassifierMixin, _BaseVoting): Parameters ---------- - estimators : list of (string, estimator) tuples + estimators : list of (str, estimator) tuples Invoking the ``fit`` method on the ``VotingClassifier`` will fit clones of those original estimators that will be stored in the class attribute ``self.estimators_``. An estimator can be set to ``'drop'`` @@ -100,44 +123,53 @@ class VotingClassifier(ClassifierMixin, _BaseVoting): Using ``None`` to drop an estimator is deprecated in 0.22 and support will be dropped in 0.24. Use the string ``'drop'`` instead. - voting : str, {'hard', 'soft'} (default='hard') + voting : {'hard', 'soft'}, default='hard' If 'hard', uses predicted class labels for majority rule voting. Else if 'soft', predicts the class label based on the argmax of the sums of the predicted probabilities, which is recommended for an ensemble of well-calibrated classifiers. - weights : array-like, shape (n_classifiers,), optional (default=`None`) + weights : array-like of shape (n_classifiers,), default=None Sequence of weights (`float` or `int`) to weight the occurrences of predicted class labels (`hard` voting) or class probabilities before averaging (`soft` voting). Uses uniform weights if `None`. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of jobs to run in parallel for ``fit``. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - flatten_transform : bool, optional (default=True) + flatten_transform : bool, default=True Affects shape of transform output only when voting='soft' If voting='soft' and flatten_transform=True, transform method returns matrix with shape (n_samples, n_classifiers * n_classes). If flatten_transform=False, it returns (n_classifiers, n_samples, n_classes). + verbose : bool, default=False + If True, the time elapsed while fitting will be printed as it + is completed. + Attributes ---------- estimators_ : list of classifiers The collection of fitted sub-estimators as defined in ``estimators`` that are not 'drop'. - named_estimators_ : Bunch object, a dictionary with attribute access + named_estimators_ : :class:`~sklearn.utils.Bunch` Attribute to access any fitted sub-estimators by name. + .. versionadded:: 0.20 - classes_ : array-like, shape (n_predictions,) + classes_ : array-like of shape (n_predictions,) The classes labels. + See Also + -------- + VotingRegressor: Prediction voting regressor. + Examples -------- >>> import numpy as np @@ -172,33 +204,30 @@ class VotingClassifier(ClassifierMixin, _BaseVoting): [1 1 1 2 2 2] >>> print(eclf3.transform(X).shape) (6, 6) - - See also - -------- - VotingRegressor: Prediction voting regressor. """ - def __init__(self, estimators, voting='hard', weights=None, n_jobs=None, - flatten_transform=True): + def __init__(self, estimators, voting='hard', weights=None, + n_jobs=None, flatten_transform=True, verbose=False): super().__init__(estimators=estimators) self.voting = voting self.weights = weights self.n_jobs = n_jobs self.flatten_transform = flatten_transform + self.verbose = verbose def fit(self, X, y, sample_weight=None): - """ Fit the estimators. + """Fit the estimators. Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training vectors, where n_samples is the number of samples and n_features is the number of features. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) Target values. - sample_weight : array-like, shape (n_samples,) or None + sample_weight : array-like of shape (n_samples,), default=None Sample weights. If None, then samples are equally weighted. Note that this is supported only if all underlying estimators support sample weights. @@ -206,6 +235,7 @@ def fit(self, X, y, sample_weight=None): Returns ------- self : object + """ check_classification_targets(y) if isinstance(y, np.ndarray) and len(y.shape) > 1 and y.shape[1] > 1: @@ -223,19 +253,18 @@ def fit(self, X, y, sample_weight=None): return super().fit(X, transformed_y, sample_weight) def predict(self, X): - """ Predict class labels for X. + """Predict class labels for X. Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Returns ------- - maj : array-like, shape (n_samples,) + maj : array-like of shape (n_samples,) Predicted class labels. """ - check_is_fitted(self) if self.voting == 'soft': maj = np.argmax(self.predict_proba(X), axis=1) @@ -252,11 +281,11 @@ def predict(self, X): return maj def _collect_probas(self, X): - """Collect results from clf.predict calls. """ + """Collect results from clf.predict calls.""" return np.asarray([clf.predict_proba(X) for clf in self.estimators_]) def _predict_proba(self, X): - """Predict class probabilities for X in 'soft' voting """ + """Predict class probabilities for X in 'soft' voting.""" check_is_fitted(self) avg = np.average(self._collect_probas(X), axis=0, weights=self._weights_not_none) @@ -268,12 +297,12 @@ def predict_proba(self): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Returns ------- - avg : array-like, shape (n_samples, n_classes) + avg : array-like of shape (n_samples, n_classes) Weighted average probability for each class per sample. """ if self.voting == 'hard': @@ -286,7 +315,7 @@ def transform(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training vectors, where n_samples is the number of samples and n_features is the number of features. @@ -294,13 +323,13 @@ def transform(self, X): ------- probabilities_or_labels If `voting='soft'` and `flatten_transform=True`: - returns array-like of shape (n_classifiers, n_samples * + returns ndarray of shape (n_classifiers, n_samples * n_classes), being class probabilities calculated by each classifier. If `voting='soft' and `flatten_transform=False`: - array-like of shape (n_classifiers, n_samples, n_classes) + ndarray of shape (n_classifiers, n_samples, n_classes) If `voting='hard'`: - array-like of shape (n_samples, n_classifiers), being + ndarray of shape (n_samples, n_classifiers), being class labels predicted by each classifier. """ check_is_fitted(self) @@ -328,7 +357,7 @@ class VotingRegressor(RegressorMixin, _BaseVoting): Parameters ---------- - estimators : list of (string, estimator) tuples + estimators : list of (str, estimator) tuples Invoking the ``fit`` method on the ``VotingRegressor`` will fit clones of those original estimators that will be stored in the class attribute ``self.estimators_``. An estimator can be set to ``'drop'`` using @@ -338,27 +367,35 @@ class VotingRegressor(RegressorMixin, _BaseVoting): Using ``None`` to drop an estimator is deprecated in 0.22 and support will be dropped in 0.24. Use the string ``'drop'`` instead. - weights : array-like, shape (n_regressors,), optional (default=`None`) + weights : array-like of shape (n_regressors,), default=None Sequence of weights (`float` or `int`) to weight the occurrences of predicted values before averaging. Uses uniform weights if `None`. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of jobs to run in parallel for ``fit``. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. + verbose : bool, default=False + If True, the time elapsed while fitting will be printed as it + is completed. + Attributes ---------- estimators_ : list of regressors The collection of fitted sub-estimators as defined in ``estimators`` that are not 'drop'. - named_estimators_ : Bunch object, a dictionary with attribute access + named_estimators_ : Bunch Attribute to access any fitted sub-estimators by name. .. versionadded:: 0.20 + See Also + -------- + VotingClassifier: Soft Voting/Majority Rule classifier. + Examples -------- >>> import numpy as np @@ -372,30 +409,27 @@ class VotingRegressor(RegressorMixin, _BaseVoting): >>> er = VotingRegressor([('lr', r1), ('rf', r2)]) >>> print(er.fit(X, y).predict(X)) [ 3.3 5.7 11.8 19.7 28. 40.3] - - See also - -------- - VotingClassifier: Soft Voting/Majority Rule classifier. """ - def __init__(self, estimators, weights=None, n_jobs=None): + def __init__(self, estimators, weights=None, n_jobs=None, verbose=False): super().__init__(estimators=estimators) self.weights = weights self.n_jobs = n_jobs + self.verbose = verbose def fit(self, X, y, sample_weight=None): - """ Fit the estimators. + """Fit the estimators. Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training vectors, where n_samples is the number of samples and n_features is the number of features. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) Target values. - sample_weight : array-like, shape (n_samples,) or None + sample_weight : array-like of shape (n_samples,), default=None Sample weights. If None, then samples are equally weighted. Note that this is supported only if all underlying estimators support sample weights. @@ -403,6 +437,7 @@ def fit(self, X, y, sample_weight=None): Returns ------- self : object + Fitted estimator. """ y = column_or_1d(y, warn=True) return super().fit(X, y, sample_weight) @@ -420,7 +455,7 @@ def predict(self, X): Returns ------- - y : array of shape (n_samples,) + y : ndarray of shape (n_samples,) The predicted values. """ check_is_fitted(self) @@ -432,14 +467,13 @@ def transform(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Returns ------- - predictions - array-like of shape (n_samples, n_classifiers), being - values predicted by each regressor. + predictions: ndarray of shape (n_samples, n_classifiers) + Values predicted by each regressor. """ check_is_fitted(self) return self._predict(X) diff --git a/sklearn/ensemble/_weight_boosting.py b/sklearn/ensemble/_weight_boosting.py index d76e29fb37239..de73858f4bb3f 100644 --- a/sklearn/ensemble/_weight_boosting.py +++ b/sklearn/ensemble/_weight_boosting.py @@ -71,25 +71,9 @@ def __init__(self, self.learning_rate = learning_rate self.random_state = random_state - def _validate_data(self, X, y=None): - - # Accept or convert to these sparse matrix formats so we can - # use _safe_indexing - accept_sparse = ['csr', 'csc'] - if y is None: - ret = check_array(X, - accept_sparse=accept_sparse, - ensure_2d=False, - allow_nd=True, - dtype=None) - else: - ret = check_X_y(X, y, - accept_sparse=accept_sparse, - ensure_2d=False, - allow_nd=True, - dtype=None, - y_numeric=is_regressor(self)) - return ret + def _check_X(self, X): + return check_array(X, accept_sparse=['csr', 'csc'], ensure_2d=True, + allow_nd=True, dtype=None) def fit(self, X, y, sample_weight=None): """Build a boosted classifier/regressor from the training set (X, y). @@ -116,7 +100,12 @@ def fit(self, X, y, sample_weight=None): if self.learning_rate <= 0: raise ValueError("learning_rate must be greater than zero") - X, y = self._validate_data(X, y) + X, y = self._validate_data(X, y, + accept_sparse=['csr', 'csc'], + ensure_2d=True, + allow_nd=True, + dtype=None, + y_numeric=is_regressor(self)) sample_weight = _check_sample_weight(sample_weight, X, np.float64) sample_weight /= sample_weight.sum() @@ -131,6 +120,8 @@ def fit(self, X, y, sample_weight=None): self.estimator_weights_ = np.zeros(self.n_estimators, dtype=np.float64) self.estimator_errors_ = np.ones(self.n_estimators, dtype=np.float64) + # Initializion of the random number instance that will be used to + # generate a seed at each iteration random_state = check_random_state(self.random_state) for iboost in range(self.n_estimators): @@ -224,10 +215,10 @@ def staged_score(self, X, y, sample_weight=None): Sample weights. Yields - ------- + ------ z : float """ - X = self._validate_data(X) + X = self._check_X(X) for y_pred in self.staged_predict(X): if is_classifier(self): @@ -237,8 +228,16 @@ def staged_score(self, X, y, sample_weight=None): @property def feature_importances_(self): - """Return the feature importances (the higher, the more important the - feature). + """The impurity-based feature importances. + + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. Returns ------- @@ -299,33 +298,34 @@ class AdaBoostClassifier(ClassifierMixin, BaseWeightBoosting): Parameters ---------- - base_estimator : object, optional (default=None) + base_estimator : object, default=None The base estimator from which the boosted ensemble is built. Support for sample weighting is required, as well as proper ``classes_`` and ``n_classes_`` attributes. If ``None``, then the base estimator is ``DecisionTreeClassifier(max_depth=1)``. - n_estimators : int, optional (default=50) + n_estimators : int, default=50 The maximum number of estimators at which boosting is terminated. In case of perfect fit, the learning procedure is stopped early. - learning_rate : float, optional (default=1.) + learning_rate : float, default=1. Learning rate shrinks the contribution of each classifier by ``learning_rate``. There is a trade-off between ``learning_rate`` and ``n_estimators``. - algorithm : {'SAMME', 'SAMME.R'}, optional (default='SAMME.R') + algorithm : {'SAMME', 'SAMME.R'}, default='SAMME.R' If 'SAMME.R' then use the SAMME.R real boosting algorithm. ``base_estimator`` must support calculation of class probabilities. If 'SAMME' then use the SAMME discrete boosting algorithm. The SAMME.R algorithm typically converges faster than SAMME, achieving a lower test error with fewer boosting iterations. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState, default=None + Controls the random seed given at each `base_estimator` at each + boosting iteration. + Thus, it is only used when `base_estimator` exposes a `random_state`. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Attributes ---------- @@ -335,21 +335,26 @@ class AdaBoostClassifier(ClassifierMixin, BaseWeightBoosting): estimators_ : list of classifiers The collection of fitted sub-estimators. - classes_ : array of shape (n_classes,) + classes_ : ndarray of shape (n_classes,) The classes labels. n_classes_ : int The number of classes. - estimator_weights_ : array of floats + estimator_weights_ : ndarray of floats Weights for each estimator in the boosted ensemble. - estimator_errors_ : array of floats + estimator_errors_ : ndarray of floats Classification error for each estimator in the boosted ensemble. feature_importances_ : ndarray of shape (n_features,) - The feature importances if supported by the ``base_estimator``. + The impurity-based feature importances if supported by the + ``base_estimator`` (when based on decision trees). + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. See Also -------- @@ -387,8 +392,6 @@ class AdaBoostClassifier(ClassifierMixin, BaseWeightBoosting): >>> clf = AdaBoostClassifier(n_estimators=100, random_state=0) >>> clf.fit(X, y) AdaBoostClassifier(n_estimators=100, random_state=0) - >>> clf.feature_importances_ - array([0.28..., 0.42..., 0.14..., 0.16...]) >>> clf.predict([[0, 0, 0, 0]]) array([1]) >>> clf.score(X, y) @@ -428,7 +431,7 @@ def fit(self, X, y, sample_weight=None): Returns ------- self : object - A fitted estimator. + Fitted estimator. """ # Check that algorithm is supported if self.algorithm not in ('SAMME', 'SAMME.R'): @@ -477,7 +480,8 @@ def _boost(self, iboost, X, y, sample_weight, random_state): The current sample weights. random_state : RandomState - The current random number generator + The RandomState instance used if the base estimator accepts a + `random_state` attribute. Returns ------- @@ -622,7 +626,7 @@ def predict(self, X): y : ndarray of shape (n_samples,) The predicted classes. """ - X = self._validate_data(X) + X = self._check_X(X) pred = self.decision_function(X) @@ -648,11 +652,11 @@ def staged_predict(self, X): DOK, or LIL. COO, DOK, and LIL are converted to CSR. Yields - ------- - y : generator of array, shape = [n_samples] + ------ + y : generator of ndarray of shape (n_samples,) The predicted classes. """ - X = self._validate_data(X) + X = self._check_X(X) n_classes = self.n_classes_ classes = self.classes_ @@ -677,7 +681,7 @@ def decision_function(self, X): Returns ------- - score : array, shape = [n_samples, k] + score : ndarray of shape of (n_samples, k) The decision function of the input samples. The order of outputs is the same of that of the :term:`classes_` attribute. Binary classification is a special cases with ``k == 1``, @@ -686,7 +690,7 @@ def decision_function(self, X): class in ``classes_``, respectively. """ check_is_fitted(self) - X = self._validate_data(X) + X = self._check_X(X) n_classes = self.n_classes_ classes = self.classes_[:, np.newaxis] @@ -719,8 +723,8 @@ def staged_decision_function(self, X): DOK, or LIL. COO, DOK, and LIL are converted to CSR. Yields - ------- - score : generator of array, shape = [n_samples, k] + ------ + score : generator of ndarray of shape (n_samples, k) The decision function of the input samples. The order of outputs is the same of that of the :term:`classes_` attribute. Binary classification is a special cases with ``k == 1``, @@ -729,7 +733,7 @@ def staged_decision_function(self, X): class in ``classes_``, respectively. """ check_is_fitted(self) - X = self._validate_data(X) + X = self._check_X(X) n_classes = self.n_classes_ classes = self.classes_[:, np.newaxis] @@ -793,12 +797,12 @@ def predict_proba(self, X): Returns ------- - p : array of shape (n_samples, n_classes) + p : ndarray of shape (n_samples, n_classes) The class probabilities of the input samples. The order of outputs is the same of that of the :term:`classes_` attribute. """ check_is_fitted(self) - X = self._validate_data(X) + X = self._check_X(X) n_classes = self.n_classes_ @@ -828,11 +832,11 @@ def staged_predict_proba(self, X): Yields ------- - p : generator of array, shape = [n_samples] + p : generator of ndarray of shape (n_samples,) The class probabilities of the input samples. The order of outputs is the same of that of the :term:`classes_` attribute. """ - X = self._validate_data(X) + X = self._check_X(X) n_classes = self.n_classes_ @@ -854,11 +858,11 @@ def predict_log_proba(self, X): Returns ------- - p : array of shape (n_samples, n_classes) + p : ndarray of shape (n_samples, n_classes) The class probabilities of the input samples. The order of outputs is the same of that of the :term:`classes_` attribute. """ - X = self._validate_data(X) + X = self._check_X(X) return np.log(self.predict_proba(X)) @@ -879,29 +883,32 @@ class AdaBoostRegressor(RegressorMixin, BaseWeightBoosting): Parameters ---------- - base_estimator : object, optional (default=None) + base_estimator : object, default=None The base estimator from which the boosted ensemble is built. If ``None``, then the base estimator is ``DecisionTreeRegressor(max_depth=3)``. - n_estimators : integer, optional (default=50) + n_estimators : int, default=50 The maximum number of estimators at which boosting is terminated. In case of perfect fit, the learning procedure is stopped early. - learning_rate : float, optional (default=1.) + learning_rate : float, default=1. Learning rate shrinks the contribution of each regressor by ``learning_rate``. There is a trade-off between ``learning_rate`` and ``n_estimators``. - loss : {'linear', 'square', 'exponential'}, optional (default='linear') + loss : {'linear', 'square', 'exponential'}, default='linear' The loss function to use when updating the weights after each boosting iteration. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState, default=None + Controls the random seed given at each `base_estimator` at each + boosting iteration. + Thus, it is only used when `base_estimator` exposes a `random_state`. + In addition, it controls the bootstrap of the weights used to train the + `base_estimator` at each boosting iteration. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Attributes ---------- @@ -911,14 +918,19 @@ class AdaBoostRegressor(RegressorMixin, BaseWeightBoosting): estimators_ : list of classifiers The collection of fitted sub-estimators. - estimator_weights_ : array of floats + estimator_weights_ : ndarray of floats Weights for each estimator in the boosted ensemble. - estimator_errors_ : array of floats + estimator_errors_ : ndarray of floats Regression error for each estimator in the boosted ensemble. feature_importances_ : ndarray of shape (n_features,) - The feature importances if supported by the ``base_estimator``. + The impurity-based feature importances if supported by the + ``base_estimator`` (when based on decision trees). + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. Examples -------- @@ -929,8 +941,6 @@ class AdaBoostRegressor(RegressorMixin, BaseWeightBoosting): >>> regr = AdaBoostRegressor(random_state=0, n_estimators=100) >>> regr.fit(X, y) AdaBoostRegressor(n_estimators=100, random_state=0) - >>> regr.feature_importances_ - array([0.2788..., 0.7109..., 0.0065..., 0.0036...]) >>> regr.predict([[0, 0, 0, 0]]) array([4.7972...]) >>> regr.score(X, y) @@ -1020,7 +1030,11 @@ def _boost(self, iboost, X, y, sample_weight, random_state): The current sample weights. random_state : RandomState - The current random number generator + The RandomState instance used if the base estimator accepts a + `random_state` attribute. + Controls also the bootstrap of the weights used to train the weak + learner. + replacement. Returns ------- @@ -1126,7 +1140,7 @@ def predict(self, X): The predicted regression values. """ check_is_fitted(self) - X = self._validate_data(X) + X = self._check_X(X) return self._get_median_predict(X, len(self.estimators_)) @@ -1147,11 +1161,11 @@ def staged_predict(self, X): Yields ------- - y : generator of array, shape = [n_samples] + y : generator of ndarray of shape (n_samples,) The predicted regression values. """ check_is_fitted(self) - X = self._validate_data(X) + X = self._check_X(X) for i, _ in enumerate(self.estimators_, 1): yield self._get_median_predict(X, limit=i) diff --git a/sklearn/ensemble/partial_dependence.py b/sklearn/ensemble/partial_dependence.py deleted file mode 100644 index 07f3eba29a136..0000000000000 --- a/sklearn/ensemble/partial_dependence.py +++ /dev/null @@ -1,441 +0,0 @@ -"""Partial dependence plots for tree ensembles. """ - -# Authors: Peter Prettenhofer -# License: BSD 3 clause - -# Note: function here are deprecated. We don't call the new versions because -# the API slightly changes (namely partial_dependence does not have the grid -# parameter anymore.) - -from itertools import count -import numbers - -import numpy as np -from scipy.stats.mstats import mquantiles -from joblib import Parallel, delayed - -from ..utils.extmath import cartesian -from ..utils import check_array -from ..utils.validation import check_is_fitted -from ..tree._tree import DTYPE -from ..utils import deprecated - -from ._gb import BaseGradientBoosting - - -__all__ = [ - 'partial_dependence', - 'plot_partial_dependence', -] - - -def _grid_from_X(X, percentiles=(0.05, 0.95), grid_resolution=100): - """Generate a grid of points based on the ``percentiles of ``X``. - - The grid is generated by placing ``grid_resolution`` equally - spaced points between the ``percentiles`` of each column - of ``X``. - - Parameters - ---------- - X : ndarray - The data - percentiles : tuple of floats - The percentiles which are used to construct the extreme - values of the grid axes. - grid_resolution : int - The number of equally spaced points that are placed - on the grid. - - Returns - ------- - grid : ndarray - All data points on the grid; ``grid.shape[1] == X.shape[1]`` - and ``grid.shape[0] == grid_resolution * X.shape[1]``. - axes : seq of ndarray - The axes with which the grid has been created. - """ - if len(percentiles) != 2: - raise ValueError('percentile must be tuple of len 2') - if not all(0. <= x <= 1. for x in percentiles): - raise ValueError('percentile values must be in [0, 1]') - - axes = [] - emp_percentiles = mquantiles(X, prob=percentiles, axis=0) - for col in range(X.shape[1]): - uniques = np.unique(X[:, col]) - if uniques.shape[0] < grid_resolution: - # feature has low resolution use unique vals - axis = uniques - else: - # create axis based on percentiles and grid resolution - axis = np.linspace(emp_percentiles[0, col], - emp_percentiles[1, col], - num=grid_resolution, endpoint=True) - axes.append(axis) - - return cartesian(axes), axes - - -@deprecated("The function ensemble.partial_dependence has been deprecated " - "in favour of inspection.partial_dependence in 0.21 " - "and will be removed in 0.23.") -def partial_dependence(gbrt, target_variables, grid=None, X=None, - percentiles=(0.05, 0.95), grid_resolution=100): - """Partial dependence of ``target_variables``. - - Partial dependence plots show the dependence between the joint values - of the ``target_variables`` and the function represented - by the ``gbrt``. - - Read more in the :ref:`User Guide `. - - .. deprecated:: 0.21 - This function was deprecated in version 0.21 in favor of - :func:`sklearn.inspection.partial_dependence` and will be - removed in 0.23. - - Parameters - ---------- - gbrt : BaseGradientBoosting - A fitted gradient boosting model. - - target_variables : array-like, dtype=int - The target features for which the partial dependency should be - computed (size should be smaller than 3 for visual renderings). - - grid : array-like of shape (n_points, n_target_variables) - The grid of ``target_variables`` values for which the - partial dependency should be evaluated (either ``grid`` or ``X`` - must be specified). - - X : array-like of shape (n_samples, n_features) - The data on which ``gbrt`` was trained. It is used to generate - a ``grid`` for the ``target_variables``. The ``grid`` comprises - ``grid_resolution`` equally spaced points between the two - ``percentiles``. - - percentiles : (low, high), default=(0.05, 0.95) - The lower and upper percentile used create the extreme values - for the ``grid``. Only if ``X`` is not None. - - grid_resolution : int, default=100 - The number of equally spaced points on the ``grid``. - - Returns - ------- - pdp : array, shape=(n_classes, n_points) - The partial dependence function evaluated on the ``grid``. - For regression and binary classification ``n_classes==1``. - - axes : seq of ndarray or None - The axes with which the grid has been created or None if - the grid has been given. - - Examples - -------- - >>> samples = [[0, 0, 2], [1, 0, 0]] - >>> labels = [0, 1] - >>> from sklearn.ensemble import GradientBoostingClassifier - >>> gb = GradientBoostingClassifier(random_state=0).fit(samples, labels) - >>> kwargs = dict(X=samples, percentiles=(0, 1), grid_resolution=2) - >>> partial_dependence(gb, [0], **kwargs) # doctest: +SKIP - (array([[-4.52..., 4.52...]]), [array([ 0., 1.])]) - """ - if not isinstance(gbrt, BaseGradientBoosting): - raise ValueError('gbrt has to be an instance of BaseGradientBoosting') - check_is_fitted(gbrt) - if (grid is None and X is None) or (grid is not None and X is not None): - raise ValueError('Either grid or X must be specified') - - target_variables = np.asarray(target_variables, dtype=np.int32, - order='C').ravel() - - if any([not (0 <= fx < gbrt.n_features_) for fx in target_variables]): - raise ValueError('target_variables must be in [0, %d]' - % (gbrt.n_features_ - 1)) - - if X is not None: - X = check_array(X, dtype=DTYPE, order='C') - grid, axes = _grid_from_X(X[:, target_variables], percentiles, - grid_resolution) - else: - assert grid is not None - # dont return axes if grid is given - axes = None - # grid must be 2d - if grid.ndim == 1: - grid = grid[:, np.newaxis] - if grid.ndim != 2: - raise ValueError('grid must be 2d but is %dd' % grid.ndim) - - grid = np.asarray(grid, dtype=DTYPE, order='C') - assert grid.shape[1] == target_variables.shape[0] - - n_trees_per_stage = gbrt.estimators_.shape[1] - n_estimators = gbrt.estimators_.shape[0] - pdp = np.zeros((n_trees_per_stage, grid.shape[0],), dtype=np.float64, - order='C') - for stage in range(n_estimators): - for k in range(n_trees_per_stage): - tree = gbrt.estimators_[stage, k].tree_ - tree.compute_partial_dependence(grid, target_variables, pdp[k]) - pdp *= gbrt.learning_rate - - return pdp, axes - - -@deprecated("The function ensemble.plot_partial_dependence has been " - "deprecated in favour of " - "sklearn.inspection.plot_partial_dependence in " - " 0.21 and will be removed in 0.23.") -def plot_partial_dependence(gbrt, X, features, feature_names=None, - label=None, n_cols=3, grid_resolution=100, - percentiles=(0.05, 0.95), n_jobs=None, - verbose=0, ax=None, line_kw=None, - contour_kw=None, **fig_kw): - """Partial dependence plots for ``features``. - - The ``len(features)`` plots are arranged in a grid with ``n_cols`` - columns. Two-way partial dependence plots are plotted as contour - plots. - - Read more in the :ref:`User Guide `. - - .. deprecated:: 0.21 - This function was deprecated in version 0.21 in favor of - :func:`sklearn.inspection.plot_partial_dependence` and will be - removed in 0.23. - - Parameters - ---------- - gbrt : BaseGradientBoosting - A fitted gradient boosting model. - - X : array-like of shape (n_samples, n_features) - The data on which ``gbrt`` was trained. - - features : seq of ints, strings, or tuples of ints or strings - If seq[i] is an int or a tuple with one int value, a one-way - PDP is created; if seq[i] is a tuple of two ints, a two-way - PDP is created. - If feature_names is specified and seq[i] is an int, seq[i] - must be < len(feature_names). - If seq[i] is a string, feature_names must be specified, and - seq[i] must be in feature_names. - - feature_names : seq of str - Name of each feature; feature_names[i] holds - the name of the feature with index i. - - label : object - The class label for which the PDPs should be computed. - Only if gbrt is a multi-class model. Must be in ``gbrt.classes_``. - - n_cols : int - The number of columns in the grid plot (default: 3). - - grid_resolution : int, default=100 - The number of equally spaced points on the axes. - - percentiles : (low, high), default=(0.05, 0.95) - The lower and upper percentile used to create the extreme values - for the PDP axes. - - n_jobs : int or None, optional (default=None) - ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. - ``-1`` means using all processors. See :term:`Glossary ` - for more details. - - verbose : int - Verbose output during PD computations. Defaults to 0. - - ax : Matplotlib axis object, default None - An axis object onto which the plots will be drawn. - - line_kw : dict - Dict with keywords passed to the ``matplotlib.pyplot.plot`` call. - For one-way partial dependence plots. - - contour_kw : dict - Dict with keywords passed to the ``matplotlib.pyplot.plot`` call. - For two-way partial dependence plots. - - ``**fig_kw`` : dict - Dict with keywords passed to the figure() call. - Note that all keywords not recognized above will be automatically - included here. - - Returns - ------- - fig : figure - The Matplotlib Figure object. - - axs : seq of Axis objects - A seq of Axis objects, one for each subplot. - - Examples - -------- - >>> from sklearn.datasets import make_friedman1 - >>> from sklearn.ensemble import GradientBoostingRegressor - >>> X, y = make_friedman1() - >>> clf = GradientBoostingRegressor(n_estimators=10).fit(X, y) - >>> fig, axs = plot_partial_dependence(clf, X, [0, (0, 1)]) #doctest: +SKIP - ... - """ - import matplotlib.pyplot as plt - from matplotlib import transforms - from matplotlib.ticker import MaxNLocator - from matplotlib.ticker import ScalarFormatter - - if not isinstance(gbrt, BaseGradientBoosting): - raise ValueError('gbrt has to be an instance of BaseGradientBoosting') - check_is_fitted(gbrt) - - # set label_idx for multi-class GBRT - if hasattr(gbrt, 'classes_') and np.size(gbrt.classes_) > 2: - if label is None: - raise ValueError('label is not given for multi-class PDP') - label_idx = np.searchsorted(gbrt.classes_, label) - if gbrt.classes_[label_idx] != label: - raise ValueError('label %s not in ``gbrt.classes_``' % str(label)) - else: - # regression and binary classification - label_idx = 0 - - X = check_array(X, dtype=DTYPE, order='C') - if gbrt.n_features_ != X.shape[1]: - raise ValueError('X.shape[1] does not match gbrt.n_features_') - - if line_kw is None: - line_kw = {'color': 'green'} - if contour_kw is None: - contour_kw = {} - - # convert feature_names to list - if feature_names is None: - # if not feature_names use fx indices as name - feature_names = [str(i) for i in range(gbrt.n_features_)] - elif isinstance(feature_names, np.ndarray): - feature_names = feature_names.tolist() - - def convert_feature(fx): - if isinstance(fx, str): - try: - fx = feature_names.index(fx) - except ValueError: - raise ValueError('Feature %s not in feature_names' % fx) - return fx - - # convert features into a seq of int tuples - tmp_features = [] - for fxs in features: - if isinstance(fxs, (numbers.Integral, str)): - fxs = (fxs,) - try: - fxs = np.array([convert_feature(fx) for fx in fxs], dtype=np.int32) - except TypeError: - raise ValueError('features must be either int, str, or tuple ' - 'of int/str') - if not (1 <= np.size(fxs) <= 2): - raise ValueError('target features must be either one or two') - - tmp_features.append(fxs) - - features = tmp_features - - names = [] - try: - for fxs in features: - l = [] - # explicit loop so "i" is bound for exception below - for i in fxs: - l.append(feature_names[i]) - names.append(l) - except IndexError: - raise ValueError('All entries of features must be less than ' - 'len(feature_names) = {0}, got {1}.' - .format(len(feature_names), i)) - - # compute PD functions - pd_result = Parallel(n_jobs=n_jobs, verbose=verbose)( - delayed(partial_dependence)(gbrt, fxs, X=X, - grid_resolution=grid_resolution, - percentiles=percentiles) - for fxs in features) - - # get global min and max values of PD grouped by plot type - pdp_lim = {} - for pdp, axes in pd_result: - min_pd, max_pd = pdp[label_idx].min(), pdp[label_idx].max() - n_fx = len(axes) - old_min_pd, old_max_pd = pdp_lim.get(n_fx, (min_pd, max_pd)) - min_pd = min(min_pd, old_min_pd) - max_pd = max(max_pd, old_max_pd) - pdp_lim[n_fx] = (min_pd, max_pd) - - # create contour levels for two-way plots - if 2 in pdp_lim: - Z_level = np.linspace(*pdp_lim[2], num=8) - - if ax is None: - fig = plt.figure(**fig_kw) - else: - fig = ax.get_figure() - fig.clear() - - n_cols = min(n_cols, len(features)) - n_rows = int(np.ceil(len(features) / float(n_cols))) - axs = [] - for i, fx, name, (pdp, axes) in zip(count(), features, names, - pd_result): - ax = fig.add_subplot(n_rows, n_cols, i + 1) - - if len(axes) == 1: - ax.plot(axes[0], pdp[label_idx].ravel(), **line_kw) - else: - # make contour plot - assert len(axes) == 2 - XX, YY = np.meshgrid(axes[0], axes[1]) - Z = pdp[label_idx].reshape(list(map(np.size, axes))).T - CS = ax.contour(XX, YY, Z, levels=Z_level, linewidths=0.5, - colors='k') - ax.contourf(XX, YY, Z, levels=Z_level, vmax=Z_level[-1], - vmin=Z_level[0], alpha=0.75, **contour_kw) - ax.clabel(CS, fmt='%2.2f', colors='k', fontsize=10, inline=True) - - # plot data deciles + axes labels - deciles = mquantiles(X[:, fx[0]], prob=np.arange(0.1, 1.0, 0.1)) - trans = transforms.blended_transform_factory(ax.transData, - ax.transAxes) - ylim = ax.get_ylim() - ax.vlines(deciles, [0], 0.05, transform=trans, color='k') - ax.set_xlabel(name[0]) - ax.set_ylim(ylim) - - # prevent x-axis ticks from overlapping - ax.xaxis.set_major_locator(MaxNLocator(nbins=6, prune='lower')) - tick_formatter = ScalarFormatter() - tick_formatter.set_powerlimits((-3, 4)) - ax.xaxis.set_major_formatter(tick_formatter) - - if len(axes) > 1: - # two-way PDP - y-axis deciles + labels - deciles = mquantiles(X[:, fx[1]], prob=np.arange(0.1, 1.0, 0.1)) - trans = transforms.blended_transform_factory(ax.transAxes, - ax.transData) - xlim = ax.get_xlim() - ax.hlines(deciles, [0], 0.05, transform=trans, color='k') - ax.set_ylabel(name[1]) - # hline erases xlim - ax.set_xlim(xlim) - else: - ax.set_ylabel('Partial dependence') - - if len(axes) == 1: - ax.set_ylim(pdp_lim[1]) - axs.append(ax) - - fig.subplots_adjust(bottom=0.15, top=0.7, left=0.1, right=0.95, wspace=0.4, - hspace=0.3) - return fig, axs diff --git a/sklearn/ensemble/tests/test_bagging.py b/sklearn/ensemble/tests/test_bagging.py index 0e69c0c8d14be..883f0067f5e78 100644 --- a/sklearn/ensemble/tests/test_bagging.py +++ b/sklearn/ensemble/tests/test_bagging.py @@ -878,3 +878,25 @@ def test_bagging_small_max_features(): bagging = BaggingClassifier(LogisticRegression(), max_features=0.3, random_state=1) bagging.fit(X, y) + + +def test_bagging_get_estimators_indices(): + # Check that Bagging estimator can generate sample indices properly + # Non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/16436 + + rng = np.random.RandomState(0) + X = rng.randn(13, 4) + y = np.arange(13) + + class MyEstimator(DecisionTreeRegressor): + """An estimator which stores y indices information at fit.""" + def fit(self, X, y): + self._sample_indices = y + + clf = BaggingRegressor(base_estimator=MyEstimator(), + n_estimators=1, random_state=0) + clf.fit(X, y) + + assert_array_equal(clf.estimators_[0]._sample_indices, + clf.estimators_samples_[0]) diff --git a/sklearn/ensemble/tests/test_forest.py b/sklearn/ensemble/tests/test_forest.py index be8335e313183..9164751bac256 100644 --- a/sklearn/ensemble/tests/test_forest.py +++ b/sklearn/ensemble/tests/test_forest.py @@ -1336,6 +1336,15 @@ def test_max_samples_exceptions(name, max_samples, exc_type, exc_msg): est.fit(X, y) +def test_forest_y_sparse(): + X = [[1, 2, 3]] + y = csr_matrix([4, 5, 6]) + est = RandomForestClassifier() + msg = "sparse multilabel-indicator for y is not supported." + with pytest.raises(ValueError, match=msg): + est.fit(X, y) + + @pytest.mark.parametrize( 'ForestClass', [RandomForestClassifier, RandomForestRegressor] ) diff --git a/sklearn/ensemble/tests/test_gradient_boosting.py b/sklearn/ensemble/tests/test_gradient_boosting.py index d0100a1724a52..0c7f07929e370 100644 --- a/sklearn/ensemble/tests/test_gradient_boosting.py +++ b/sklearn/ensemble/tests/test_gradient_boosting.py @@ -345,10 +345,6 @@ def test_check_inputs(): clf = GradientBoostingClassifier(n_estimators=100, random_state=1) assert_raises(ValueError, clf.fit, X, y + [0, 1]) - clf = GradientBoostingClassifier(n_estimators=100, random_state=1) - assert_raises(ValueError, clf.fit, X, y, - sample_weight=([1] * len(y)) + [0, 1]) - weight = [0, 0, 0, 1, 1, 1] clf = GradientBoostingClassifier(n_estimators=100, random_state=1) msg = ("y contains 1 class after sample_weight trimmed classes with " @@ -1170,9 +1166,10 @@ def test_non_uniform_weights_toy_edge_case_clf(): def check_sparse_input(EstimatorClass, X, X_sparse, y): dense = EstimatorClass(n_estimators=10, random_state=0, - max_depth=2).fit(X, y) + max_depth=2, min_impurity_decrease=1e-7).fit(X, y) sparse = EstimatorClass(n_estimators=10, random_state=0, - max_depth=2).fit(X_sparse, y) + max_depth=2, + min_impurity_decrease=1e-7).fit(X_sparse, y) assert_array_almost_equal(sparse.apply(X), dense.apply(X)) assert_array_almost_equal(sparse.predict(X), dense.predict(X)) @@ -1311,7 +1308,7 @@ def test_gradient_boosting_with_init(gb, dataset_maker, init_estimator): # Check that GradientBoostingRegressor works when init is a sklearn # estimator. # Check that an error is raised if trying to fit with sample weight but - # inital estimator does not support sample weight + # initial estimator does not support sample weight X, y = dataset_maker() sample_weight = np.random.RandomState(42).rand(100) diff --git a/sklearn/ensemble/tests/test_partial_dependence.py b/sklearn/ensemble/tests/test_partial_dependence.py deleted file mode 100644 index a56523a29836c..0000000000000 --- a/sklearn/ensemble/tests/test_partial_dependence.py +++ /dev/null @@ -1,277 +0,0 @@ -""" -Testing for the partial dependence module. -""" -import pytest - -import numpy as np -from numpy.testing import assert_array_equal, assert_allclose - -from sklearn.utils._testing import assert_raises -from sklearn.ensemble.partial_dependence import partial_dependence -from sklearn.ensemble.partial_dependence import plot_partial_dependence -from sklearn.ensemble import GradientBoostingClassifier -from sklearn.ensemble import GradientBoostingRegressor -from sklearn import datasets -from sklearn.utils._testing import ignore_warnings - - -# toy sample -X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]] -y = [-1, -1, -1, 1, 1, 1] -sample_weight = [1, 1, 1, 2, 2, 2] - -# also load the boston dataset -boston = datasets.load_boston() - -# also load the iris dataset -iris = datasets.load_iris() - - -@ignore_warnings(category=FutureWarning) -def test_partial_dependence_classifier(): - # Test partial dependence for classifier - clf = GradientBoostingClassifier(n_estimators=10, random_state=1) - clf.fit(X, y) - - pdp, axes = partial_dependence(clf, [0], X=X, grid_resolution=5) - - # only 4 grid points instead of 5 because only 4 unique X[:,0] vals - assert pdp.shape == (1, 4) - assert axes[0].shape[0] == 4 - - # now with our own grid - X_ = np.asarray(X) - grid = np.unique(X_[:, 0]) - pdp_2, axes = partial_dependence(clf, [0], grid=grid) - - assert axes is None - assert_array_equal(pdp, pdp_2) - - # with trivial (no-op) sample weights - clf.fit(X, y, sample_weight=np.ones(len(y))) - - pdp_w, axes_w = partial_dependence(clf, [0], X=X, grid_resolution=5) - - assert pdp_w.shape == (1, 4) - assert axes_w[0].shape[0] == 4 - assert_allclose(pdp_w, pdp) - - # with non-trivial sample weights - clf.fit(X, y, sample_weight=sample_weight) - - pdp_w2, axes_w2 = partial_dependence(clf, [0], X=X, grid_resolution=5) - - assert pdp_w2.shape == (1, 4) - assert axes_w2[0].shape[0] == 4 - assert np.all(np.abs(pdp_w2 - pdp_w) / np.abs(pdp_w) > 0.1) - - -@ignore_warnings(category=FutureWarning) -def test_partial_dependence_multiclass(): - # Test partial dependence for multi-class classifier - clf = GradientBoostingClassifier(n_estimators=10, random_state=1) - clf.fit(iris.data, iris.target) - - grid_resolution = 25 - n_classes = clf.n_classes_ - pdp, axes = partial_dependence( - clf, [0], X=iris.data, grid_resolution=grid_resolution) - - assert pdp.shape == (n_classes, grid_resolution) - assert len(axes) == 1 - assert axes[0].shape[0] == grid_resolution - - -@ignore_warnings(category=FutureWarning) -def test_partial_dependence_regressor(): - # Test partial dependence for regressor - clf = GradientBoostingRegressor(n_estimators=10, random_state=1) - clf.fit(boston.data, boston.target) - - grid_resolution = 25 - pdp, axes = partial_dependence( - clf, [0], X=boston.data, grid_resolution=grid_resolution) - - assert pdp.shape == (1, grid_resolution) - assert axes[0].shape[0] == grid_resolution - - -@ignore_warnings(category=FutureWarning) -def test_partial_dependence_sample_weight(): - # Test near perfect correlation between partial dependence and diagonal - # when sample weights emphasize y = x predictions - N = 1000 - rng = np.random.RandomState(123456) - mask = rng.randint(2, size=N, dtype=bool) - - x = rng.rand(N) - # set y = x on mask and y = -x outside - y = x.copy() - y[~mask] = -y[~mask] - X = np.c_[mask, x] - # sample weights to emphasize data points where y = x - sample_weight = np.ones(N) - sample_weight[mask] = 1000. - - clf = GradientBoostingRegressor(n_estimators=10, random_state=1) - clf.fit(X, y, sample_weight=sample_weight) - - grid = np.arange(0, 1, 0.01) - pdp = partial_dependence(clf, [1], grid=grid) - - assert np.corrcoef(np.ravel(pdp[0]), grid)[0, 1] > 0.99 - - -@ignore_warnings(category=FutureWarning) -def test_partial_dependecy_input(): - # Test input validation of partial dependence. - clf = GradientBoostingClassifier(n_estimators=10, random_state=1) - clf.fit(X, y) - - assert_raises(ValueError, partial_dependence, - clf, [0], grid=None, X=None) - - assert_raises(ValueError, partial_dependence, - clf, [0], grid=[0, 1], X=X) - - # first argument must be an instance of BaseGradientBoosting - assert_raises(ValueError, partial_dependence, - {}, [0], X=X) - - # Gradient boosting estimator must be fit - assert_raises(ValueError, partial_dependence, - GradientBoostingClassifier(), [0], X=X) - - assert_raises(ValueError, partial_dependence, clf, [-1], X=X) - - assert_raises(ValueError, partial_dependence, clf, [100], X=X) - - # wrong ndim for grid - grid = np.random.rand(10, 2, 1) - assert_raises(ValueError, partial_dependence, clf, [0], grid=grid) - - -@ignore_warnings(category=FutureWarning) -@pytest.mark.filterwarnings('ignore: Using or importing the ABCs from') -# matplotlib Python3.7 warning -def test_plot_partial_dependence(pyplot): - # Test partial dependence plot function. - clf = GradientBoostingRegressor(n_estimators=10, random_state=1) - clf.fit(boston.data, boston.target) - - grid_resolution = 25 - fig, axs = plot_partial_dependence(clf, boston.data, [0, 1, (0, 1)], - grid_resolution=grid_resolution, - feature_names=boston.feature_names) - assert len(axs) == 3 - assert all(ax.has_data for ax in axs) - - # check with str features and array feature names - fig, axs = plot_partial_dependence(clf, boston.data, ['CRIM', 'ZN', - ('CRIM', 'ZN')], - grid_resolution=grid_resolution, - feature_names=boston.feature_names) - - assert len(axs) == 3 - assert all(ax.has_data for ax in axs) - - # check with list feature_names - feature_names = boston.feature_names.tolist() - fig, axs = plot_partial_dependence(clf, boston.data, ['CRIM', 'ZN', - ('CRIM', 'ZN')], - grid_resolution=grid_resolution, - feature_names=feature_names) - assert len(axs) == 3 - assert all(ax.has_data for ax in axs) - - -@pytest.mark.filterwarnings('ignore: Using or importing the ABCs from') -# matplotlib Python3.7 warning -@ignore_warnings(category=FutureWarning) -def test_plot_partial_dependence_input(pyplot): - # Test partial dependence plot function input checks. - clf = GradientBoostingClassifier(n_estimators=10, random_state=1) - - # not fitted yet - assert_raises(ValueError, plot_partial_dependence, - clf, X, [0]) - - clf.fit(X, y) - - assert_raises(ValueError, plot_partial_dependence, - clf, np.array(X)[:, :0], [0]) - - # first argument must be an instance of BaseGradientBoosting - assert_raises(ValueError, plot_partial_dependence, - {}, X, [0]) - - # must be larger than -1 - assert_raises(ValueError, plot_partial_dependence, - clf, X, [-1]) - - # too large feature value - assert_raises(ValueError, plot_partial_dependence, - clf, X, [100]) - - # str feature but no feature_names - assert_raises(ValueError, plot_partial_dependence, - clf, X, ['foobar']) - - # not valid features value - assert_raises(ValueError, plot_partial_dependence, - clf, X, [{'foo': 'bar'}]) - - -@pytest.mark.filterwarnings('ignore: Using or importing the ABCs from') -# matplotlib Python3.7 warning -@ignore_warnings(category=FutureWarning) -def test_plot_partial_dependence_multiclass(pyplot): - # Test partial dependence plot function on multi-class input. - clf = GradientBoostingClassifier(n_estimators=10, random_state=1) - clf.fit(iris.data, iris.target) - - grid_resolution = 25 - fig, axs = plot_partial_dependence(clf, iris.data, [0, 1], - label=0, - grid_resolution=grid_resolution) - assert len(axs) == 2 - assert all(ax.has_data for ax in axs) - - # now with symbol labels - target = iris.target_names[iris.target] - clf = GradientBoostingClassifier(n_estimators=10, random_state=1) - clf.fit(iris.data, target) - - grid_resolution = 25 - fig, axs = plot_partial_dependence(clf, iris.data, [0, 1], - label='setosa', - grid_resolution=grid_resolution) - assert len(axs) == 2 - assert all(ax.has_data for ax in axs) - - # label not in gbrt.classes_ - assert_raises(ValueError, plot_partial_dependence, - clf, iris.data, [0, 1], label='foobar', - grid_resolution=grid_resolution) - - # label not provided - assert_raises(ValueError, plot_partial_dependence, - clf, iris.data, [0, 1], - grid_resolution=grid_resolution) - - -@pytest.mark.parametrize( - "func, params", - [(partial_dependence, {'target_variables': [0], 'X': boston.data}), - (plot_partial_dependence, {'X': boston.data, 'features': [0, 1, (0, 1)]})] -) -def test_raise_deprecation_warning(pyplot, func, params): - clf = GradientBoostingRegressor(n_estimators=10, random_state=1) - clf.fit(boston.data, boston.target) - grid_resolution = 25 - - warn_msg = "The function ensemble.{} has been deprecated".format( - func.__name__ - ) - with pytest.warns(FutureWarning, match=warn_msg): - func(clf, **params, grid_resolution=grid_resolution) diff --git a/sklearn/ensemble/tests/test_stacking.py b/sklearn/ensemble/tests/test_stacking.py index 1eff7ba5f7de7..f8a3f290e96b5 100644 --- a/sklearn/ensemble/tests/test_stacking.py +++ b/sklearn/ensemble/tests/test_stacking.py @@ -38,6 +38,7 @@ from sklearn.model_selection import StratifiedKFold from sklearn.model_selection import KFold +from sklearn.utils._mocking import CheckingClassifier from sklearn.utils._testing import assert_allclose from sklearn.utils._testing import assert_allclose_dense_sparse from sklearn.utils._testing import ignore_warnings @@ -439,6 +440,19 @@ def test_stacking_with_sample_weight(stacker, X, y): assert np.abs(y_pred_no_weight - y_pred_biased).sum() > 0 +def test_stacking_classifier_sample_weight_fit_param(): + # check sample_weight is passed to all invocations of fit + stacker = StackingClassifier( + estimators=[ + ('lr', CheckingClassifier(expected_fit_params=['sample_weight'])) + ], + final_estimator=CheckingClassifier( + expected_fit_params=['sample_weight'] + ) + ) + stacker.fit(X_iris, y_iris, sample_weight=np.ones(X_iris.shape[0])) + + @pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning") @pytest.mark.parametrize( "stacker, X, y", diff --git a/sklearn/ensemble/tests/test_voting.py b/sklearn/ensemble/tests/test_voting.py index 8619296536964..4eb47bea0a514 100644 --- a/sklearn/ensemble/tests/test_voting.py +++ b/sklearn/ensemble/tests/test_voting.py @@ -1,6 +1,7 @@ """Testing for the VotingClassifier and VotingRegressor""" import pytest +import re import numpy as np from sklearn.utils._testing import assert_almost_equal, assert_array_equal @@ -315,7 +316,7 @@ def test_sample_weight(): with pytest.raises(TypeError, match=msg): eclf3.fit(X, y, sample_weight) - # check that _parallel_fit_estimator will raise the right error + # check that _fit_single_estimator will raise the right error # it should raise the original error if this is not linked to sample_weight class ClassifierErrorFit(ClassifierMixin, BaseEstimator): def fit(self, X, y, sample_weight): @@ -513,6 +514,49 @@ def test_check_estimators_voting_estimator(estimator): check_no_attributes_set_in_init(estimator.__class__.__name__, estimator) +@pytest.mark.parametrize( + "est", + [VotingRegressor( + estimators=[('lr', LinearRegression()), + ('tree', DecisionTreeRegressor(random_state=0))]), + VotingClassifier( + estimators=[('lr', LogisticRegression(random_state=0)), + ('tree', DecisionTreeClassifier(random_state=0))])], + ids=['VotingRegressor', 'VotingClassifier'] +) +def test_n_features_in(est): + + X = [[1, 2], [3, 4], [5, 6]] + y = [0, 1, 2] + + assert not hasattr(est, 'n_features_in_') + est.fit(X, y) + assert est.n_features_in_ == 2 + + +@pytest.mark.parametrize( + "estimator", + [VotingRegressor( + estimators=[('lr', LinearRegression()), + ('rf', RandomForestRegressor(random_state=123))], + verbose=True), + VotingClassifier( + estimators=[('lr', LogisticRegression(random_state=123)), + ('rf', RandomForestClassifier(random_state=123))], + verbose=True)] +) +def test_voting_verbose(estimator, capsys): + + X = np.array([[-1.1, -1.5], [-1.2, -1.4], [-3.4, -2.2], [1.1, 1.2]]) + y = np.array([1, 1, 2, 2]) + + pattern = (r'\[Voting\].*\(1 of 2\) Processing lr, total=.*\n' + r'\[Voting\].*\(2 of 2\) Processing rf, total=.*\n$') + + estimator.fit(X, y) + assert re.match(pattern, capsys.readouterr()[0]) + + # TODO: Remove in 0.24 when None is removed in Voting* @pytest.mark.parametrize( "Voter, BaseEstimator", diff --git a/sklearn/ensemble/tests/test_weight_boosting.py b/sklearn/ensemble/tests/test_weight_boosting.py index d225f2c78c2b1..c71329be9ec71 100755 --- a/sklearn/ensemble/tests/test_weight_boosting.py +++ b/sklearn/ensemble/tests/test_weight_boosting.py @@ -526,7 +526,7 @@ def test_adaboostregressor_sample_weight(): X[-1] *= 10 y[-1] = 10000 - # random_state=0 ensure that the underlying boostrap will use the outlier + # random_state=0 ensure that the underlying bootstrap will use the outlier regr_no_outlier = AdaBoostRegressor( base_estimator=LinearRegression(), n_estimators=1, random_state=0 ) diff --git a/sklearn/exceptions.py b/sklearn/exceptions.py index 95a876b88be4d..ea34365afa703 100644 --- a/sklearn/exceptions.py +++ b/sklearn/exceptions.py @@ -12,7 +12,8 @@ 'FitFailedWarning', 'NonBLASDotWarning', 'SkipTestWarning', - 'UndefinedMetricWarning'] + 'UndefinedMetricWarning', + 'PositiveSpectrumWarning'] class NotFittedError(ValueError, AttributeError): @@ -61,8 +62,8 @@ class ConvergenceWarning(UserWarning): ... [1, 0], ... [1, 0]]) # last point is duplicated >>> with warnings.catch_warnings(record=True) as w: - ... km = KMeans(n_clusters=4).fit(X) - ... print(w[-1].message) + ... km = KMeans(n_clusters=4).fit(X) + ... print(w[-1].message) Number of distinct clusters (3) found smaller than n_clusters (4). Possibly due to duplicate points in X. @@ -138,7 +139,9 @@ class FitFailedWarning(RuntimeWarning): ... print(repr(w[-1].message)) FitFailedWarning('Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. - Details: \\nValueError: Penalty term must be positive; got (C=-2)\\n'...) + Details:...Traceback (most recent call last):...ValueError: + Penalty term must be positive; got (C=-2)... + .. versionchanged:: 0.18 Moved from sklearn.cross_validation. @@ -171,3 +174,15 @@ class UndefinedMetricWarning(UserWarning): .. versionchanged:: 0.18 Moved from sklearn.base. """ + + +class PositiveSpectrumWarning(UserWarning): + """Warning raised when the eigenvalues of a PSD matrix have issues + + This warning is typically raised by ``_check_psd_eigenvalues`` when the + eigenvalues of a positive semidefinite (PSD) matrix such as a gram matrix + (kernel) present significant negative eigenvalues, or bad conditioning i.e. + very small non-zero eigenvalues compared to the largest eigenvalue. + + .. versionadded:: 0.22 + """ diff --git a/sklearn/externals/_arff.py b/sklearn/externals/_arff.py index 4db55eb6d6c02..bf3cbfc9a9b98 100644 --- a/sklearn/externals/_arff.py +++ b/sklearn/externals/_arff.py @@ -98,7 +98,7 @@ The above keys must follow the case which were described, i.e., the keys are case sensitive. The attribute type ``attribute_type`` must be one of these strings (they are not case sensitive): ``NUMERIC``, ``INTEGER``, ``REAL`` or -``STRING``. For nominal attributes, the ``atribute_type`` must be a list of +``STRING``. For nominal attributes, the ``attribute_type`` must be a list of strings. In this format, the XOR dataset presented above can be represented as a python diff --git a/sklearn/externals/joblib/numpy_pickle.py b/sklearn/externals/joblib/numpy_pickle.py index 7a4a2885c9f15..e79a0e1c5c056 100644 --- a/sklearn/externals/joblib/numpy_pickle.py +++ b/sklearn/externals/joblib/numpy_pickle.py @@ -1,3 +1,3 @@ -# Import necessary to preserve backward compatibliity of pickles +# Import necessary to preserve backward compatibility of pickles from joblib.numpy_pickle import * diff --git a/sklearn/externals/six.py b/sklearn/externals/six.py deleted file mode 100644 index 26d95f7df9abc..0000000000000 --- a/sklearn/externals/six.py +++ /dev/null @@ -1,583 +0,0 @@ -"""Utilities for writing code that runs on Python 2 and 3""" - -# Copyright (c) 2010-2013 Benjamin Peterson -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import operator -import sys -import types - -import warnings -warnings.warn("The module is deprecated in version 0.21 and will be removed " - "in version 0.23 since we've dropped support for Python 2.7. " - "Please rely on the official version of six " - "(https://pypi.org/project/six/).", FutureWarning) - -__author__ = "Benjamin Peterson " -__version__ = "1.4.1" - - -# Useful for very coarse version differentiation. -PY2 = sys.version_info[0] == 2 -PY3 = sys.version_info[0] == 3 - -if PY3: - string_types = str, - integer_types = int, - class_types = type, - text_type = str - binary_type = bytes - - MAXSIZE = sys.maxsize -else: - string_types = basestring, - integer_types = (int, long) - class_types = (type, types.ClassType) - text_type = unicode - binary_type = str - - if sys.platform.startswith("java"): - # Jython always uses 32 bits. - MAXSIZE = int((1 << 31) - 1) - else: - # It's possible to have sizeof(long) != sizeof(Py_ssize_t). - class X(object): - def __len__(self): - return 1 << 31 - try: - len(X()) - except OverflowError: - # 32-bit - MAXSIZE = int((1 << 31) - 1) - else: - # 64-bit - MAXSIZE = int((1 << 63) - 1) - del X - - -def _add_doc(func, doc): - """Add documentation to a function.""" - func.__doc__ = doc - - -def _import_module(name): - """Import module, returning the module after the last dot.""" - __import__(name) - return sys.modules[name] - - -class _LazyDescr(object): - - def __init__(self, name): - self.name = name - - def __get__(self, obj, tp): - result = self._resolve() - setattr(obj, self.name, result) - # This is a bit ugly, but it avoids running this again. - delattr(tp, self.name) - return result - - -class MovedModule(_LazyDescr): - - def __init__(self, name, old, new=None): - super(MovedModule, self).__init__(name) - if PY3: - if new is None: - new = name - self.mod = new - else: - self.mod = old - - def _resolve(self): - return _import_module(self.mod) - - -class MovedAttribute(_LazyDescr): - - def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): - super(MovedAttribute, self).__init__(name) - if PY3: - if new_mod is None: - new_mod = name - self.mod = new_mod - if new_attr is None: - if old_attr is None: - new_attr = name - else: - new_attr = old_attr - self.attr = new_attr - else: - self.mod = old_mod - if old_attr is None: - old_attr = name - self.attr = old_attr - - def _resolve(self): - module = _import_module(self.mod) - return getattr(module, self.attr) - - - -class _MovedItems(types.ModuleType): - """Lazy loading of moved objects""" - - -_moved_attributes = [ - MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), - MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), - MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), - MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), - MovedAttribute("map", "itertools", "builtins", "imap", "map"), - MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute("reload_module", "__builtin__", "imp", "reload"), - MovedAttribute("reduce", "__builtin__", "functools"), - MovedAttribute("StringIO", "StringIO", "io"), - MovedAttribute("UserString", "UserString", "collections"), - MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), - MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), - - MovedModule("builtins", "__builtin__"), - MovedModule("configparser", "ConfigParser"), - MovedModule("copyreg", "copy_reg"), - MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), - MovedModule("http_cookies", "Cookie", "http.cookies"), - MovedModule("html_entities", "htmlentitydefs", "html.entities"), - MovedModule("html_parser", "HTMLParser", "html.parser"), - MovedModule("http_client", "httplib", "http.client"), - MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), - MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), - MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), - MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), - MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), - MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), - MovedModule("cPickle", "cPickle", "pickle"), - MovedModule("queue", "Queue"), - MovedModule("reprlib", "repr"), - MovedModule("socketserver", "SocketServer"), - MovedModule("tkinter", "Tkinter"), - MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), - MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), - MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), - MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), - MovedModule("tkinter_tix", "Tix", "tkinter.tix"), - MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), - MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), - MovedModule("tkinter_colorchooser", "tkColorChooser", - "tkinter.colorchooser"), - MovedModule("tkinter_commondialog", "tkCommonDialog", - "tkinter.commondialog"), - MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), - MovedModule("tkinter_font", "tkFont", "tkinter.font"), - MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), - MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", - "tkinter.simpledialog"), - MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), - MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), - MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), - MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), - MovedModule("winreg", "_winreg"), -] -for attr in _moved_attributes: - setattr(_MovedItems, attr.name, attr) -del attr - -moves = sys.modules[__name__ + ".moves"] = _MovedItems(__name__ + ".moves") - - - -class Module_six_moves_urllib_parse(types.ModuleType): - """Lazy loading of moved objects in six.moves.urllib_parse""" - - -_urllib_parse_moved_attributes = [ - MovedAttribute("ParseResult", "urlparse", "urllib.parse"), - MovedAttribute("parse_qs", "urlparse", "urllib.parse"), - MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), - MovedAttribute("urldefrag", "urlparse", "urllib.parse"), - MovedAttribute("urljoin", "urlparse", "urllib.parse"), - MovedAttribute("urlparse", "urlparse", "urllib.parse"), - MovedAttribute("urlsplit", "urlparse", "urllib.parse"), - MovedAttribute("urlunparse", "urlparse", "urllib.parse"), - MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), - MovedAttribute("quote", "urllib", "urllib.parse"), - MovedAttribute("quote_plus", "urllib", "urllib.parse"), - MovedAttribute("unquote", "urllib", "urllib.parse"), - MovedAttribute("unquote_plus", "urllib", "urllib.parse"), - MovedAttribute("urlencode", "urllib", "urllib.parse"), -] -for attr in _urllib_parse_moved_attributes: - setattr(Module_six_moves_urllib_parse, attr.name, attr) -del attr - -sys.modules[__name__ + ".moves.urllib_parse"] = Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse") -sys.modules[__name__ + ".moves.urllib.parse"] = Module_six_moves_urllib_parse(__name__ + ".moves.urllib.parse") - - -class Module_six_moves_urllib_error(types.ModuleType): - """Lazy loading of moved objects in six.moves.urllib_error""" - - -_urllib_error_moved_attributes = [ - MovedAttribute("URLError", "urllib2", "urllib.error"), - MovedAttribute("HTTPError", "urllib2", "urllib.error"), - MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), -] -for attr in _urllib_error_moved_attributes: - setattr(Module_six_moves_urllib_error, attr.name, attr) -del attr - -sys.modules[__name__ + ".moves.urllib_error"] = Module_six_moves_urllib_error(__name__ + ".moves.urllib_error") -sys.modules[__name__ + ".moves.urllib.error"] = Module_six_moves_urllib_error(__name__ + ".moves.urllib.error") - - -class Module_six_moves_urllib_request(types.ModuleType): - """Lazy loading of moved objects in six.moves.urllib_request""" - - -_urllib_request_moved_attributes = [ - MovedAttribute("urlopen", "urllib2", "urllib.request"), - MovedAttribute("install_opener", "urllib2", "urllib.request"), - MovedAttribute("build_opener", "urllib2", "urllib.request"), - MovedAttribute("pathname2url", "urllib", "urllib.request"), - MovedAttribute("url2pathname", "urllib", "urllib.request"), - MovedAttribute("getproxies", "urllib", "urllib.request"), - MovedAttribute("Request", "urllib2", "urllib.request"), - MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), - MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), - MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), - MovedAttribute("BaseHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), - MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), - MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), - MovedAttribute("FileHandler", "urllib2", "urllib.request"), - MovedAttribute("FTPHandler", "urllib2", "urllib.request"), - MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), - MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), - MovedAttribute("urlretrieve", "urllib", "urllib.request"), - MovedAttribute("urlcleanup", "urllib", "urllib.request"), - MovedAttribute("URLopener", "urllib", "urllib.request"), - MovedAttribute("FancyURLopener", "urllib", "urllib.request"), -] -for attr in _urllib_request_moved_attributes: - setattr(Module_six_moves_urllib_request, attr.name, attr) -del attr - -sys.modules[__name__ + ".moves.urllib_request"] = Module_six_moves_urllib_request(__name__ + ".moves.urllib_request") -sys.modules[__name__ + ".moves.urllib.request"] = Module_six_moves_urllib_request(__name__ + ".moves.urllib.request") - - -class Module_six_moves_urllib_response(types.ModuleType): - """Lazy loading of moved objects in six.moves.urllib_response""" - - -_urllib_response_moved_attributes = [ - MovedAttribute("addbase", "urllib", "urllib.response"), - MovedAttribute("addclosehook", "urllib", "urllib.response"), - MovedAttribute("addinfo", "urllib", "urllib.response"), - MovedAttribute("addinfourl", "urllib", "urllib.response"), -] -for attr in _urllib_response_moved_attributes: - setattr(Module_six_moves_urllib_response, attr.name, attr) -del attr - -sys.modules[__name__ + ".moves.urllib_response"] = Module_six_moves_urllib_response(__name__ + ".moves.urllib_response") -sys.modules[__name__ + ".moves.urllib.response"] = Module_six_moves_urllib_response(__name__ + ".moves.urllib.response") - - -class Module_six_moves_urllib_robotparser(types.ModuleType): - """Lazy loading of moved objects in six.moves.urllib_robotparser""" - - -_urllib_robotparser_moved_attributes = [ - MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), -] -for attr in _urllib_robotparser_moved_attributes: - setattr(Module_six_moves_urllib_robotparser, attr.name, attr) -del attr - -sys.modules[__name__ + ".moves.urllib_robotparser"] = Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib_robotparser") -sys.modules[__name__ + ".moves.urllib.robotparser"] = Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser") - - -class Module_six_moves_urllib(types.ModuleType): - """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" - parse = sys.modules[__name__ + ".moves.urllib_parse"] - error = sys.modules[__name__ + ".moves.urllib_error"] - request = sys.modules[__name__ + ".moves.urllib_request"] - response = sys.modules[__name__ + ".moves.urllib_response"] - robotparser = sys.modules[__name__ + ".moves.urllib_robotparser"] - - -sys.modules[__name__ + ".moves.urllib"] = Module_six_moves_urllib(__name__ + ".moves.urllib") - - -def add_move(move): - """Add an item to six.moves.""" - setattr(_MovedItems, move.name, move) - - -def remove_move(name): - """Remove item from six.moves.""" - try: - delattr(_MovedItems, name) - except AttributeError: - try: - del moves.__dict__[name] - except KeyError: - raise AttributeError("no such move, %r" % (name,)) - - -if PY3: - _meth_func = "__func__" - _meth_self = "__self__" - - _func_closure = "__closure__" - _func_code = "__code__" - _func_defaults = "__defaults__" - _func_globals = "__globals__" - - _iterkeys = "keys" - _itervalues = "values" - _iteritems = "items" - _iterlists = "lists" -else: - _meth_func = "im_func" - _meth_self = "im_self" - - _func_closure = "func_closure" - _func_code = "func_code" - _func_defaults = "func_defaults" - _func_globals = "func_globals" - - _iterkeys = "iterkeys" - _itervalues = "itervalues" - _iteritems = "iteritems" - _iterlists = "iterlists" - - -try: - advance_iterator = next -except NameError: - def advance_iterator(it): - return it.next() -next = advance_iterator - - -try: - callable = callable -except NameError: - def callable(obj): - return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) - - -if PY3: - def get_unbound_function(unbound): - return unbound - - create_bound_method = types.MethodType - - Iterator = object -else: - def get_unbound_function(unbound): - return unbound.im_func - - def create_bound_method(func, obj): - return types.MethodType(func, obj, obj.__class__) - - class Iterator(object): - - def next(self): - return type(self).__next__(self) - - callable = callable -_add_doc(get_unbound_function, - """Get the function out of a possibly unbound function""") - - -get_method_function = operator.attrgetter(_meth_func) -get_method_self = operator.attrgetter(_meth_self) -get_function_closure = operator.attrgetter(_func_closure) -get_function_code = operator.attrgetter(_func_code) -get_function_defaults = operator.attrgetter(_func_defaults) -get_function_globals = operator.attrgetter(_func_globals) - - -def iterkeys(d, **kw): - """Return an iterator over the keys of a dictionary.""" - return iter(getattr(d, _iterkeys)(**kw)) - -def itervalues(d, **kw): - """Return an iterator over the values of a dictionary.""" - return iter(getattr(d, _itervalues)(**kw)) - -def iteritems(d, **kw): - """Return an iterator over the (key, value) pairs of a dictionary.""" - return iter(getattr(d, _iteritems)(**kw)) - -def iterlists(d, **kw): - """Return an iterator over the (key, [values]) pairs of a dictionary.""" - return iter(getattr(d, _iterlists)(**kw)) - - -if PY3: - def b(s): - return s.encode("latin-1") - def u(s): - return s - unichr = chr - if sys.version_info[1] <= 1: - def int2byte(i): - return bytes((i,)) - else: - # This is about 2x faster than the implementation above on 3.2+ - int2byte = operator.methodcaller("to_bytes", 1, "big") - byte2int = operator.itemgetter(0) - indexbytes = operator.getitem - iterbytes = iter - import io - StringIO = io.StringIO - BytesIO = io.BytesIO -else: - def b(s): - return s - def u(s): - return unicode(s, "unicode_escape") - unichr = unichr - int2byte = chr - def byte2int(bs): - return ord(bs[0]) - def indexbytes(buf, i): - return ord(buf[i]) - def iterbytes(buf): - return (ord(byte) for byte in buf) - import StringIO - StringIO = BytesIO = StringIO.StringIO -_add_doc(b, """Byte literal""") -_add_doc(u, """Text literal""") - - -if PY3: - import builtins - exec_ = getattr(builtins, "exec") - - - def reraise(tp, value, tb=None): - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value - - - print_ = getattr(builtins, "print") - del builtins - -else: - def exec_(_code_, _globs_=None, _locs_=None): - """Execute code in a namespace.""" - if _globs_ is None: - frame = sys._getframe(1) - _globs_ = frame.f_globals - if _locs_ is None: - _locs_ = frame.f_locals - del frame - elif _locs_ is None: - _locs_ = _globs_ - exec("""exec _code_ in _globs_, _locs_""") - - - exec_("""def reraise(tp, value, tb=None): - raise tp, value, tb -""") - - - def print_(*args, **kwargs): - """The new-style print function.""" - fp = kwargs.pop("file", sys.stdout) - if fp is None: - return - def write(data): - if not isinstance(data, basestring): - data = str(data) - fp.write(data) - want_unicode = False - sep = kwargs.pop("sep", None) - if sep is not None: - if isinstance(sep, unicode): - want_unicode = True - elif not isinstance(sep, str): - raise TypeError("sep must be None or a string") - end = kwargs.pop("end", None) - if end is not None: - if isinstance(end, unicode): - want_unicode = True - elif not isinstance(end, str): - raise TypeError("end must be None or a string") - if kwargs: - raise TypeError("invalid keyword arguments to print()") - if not want_unicode: - for arg in args: - if isinstance(arg, unicode): - want_unicode = True - break - if want_unicode: - newline = unicode("\n") - space = unicode(" ") - else: - newline = "\n" - space = " " - if sep is None: - sep = space - if end is None: - end = newline - for i, arg in enumerate(args): - if i: - write(sep) - write(arg) - write(end) - -_add_doc(reraise, """Reraise an exception.""") - - -def with_metaclass(meta, *bases): - """Create a base class with a metaclass.""" - return meta("NewBase", bases, {}) - -def add_metaclass(metaclass): - """Class decorator for creating a class with a metaclass.""" - def wrapper(cls): - orig_vars = cls.__dict__.copy() - orig_vars.pop('__dict__', None) - orig_vars.pop('__weakref__', None) - for slots_var in orig_vars.get('__slots__', ()): - orig_vars.pop(slots_var) - return metaclass(cls.__name__, cls.__bases__, orig_vars) - return wrapper diff --git a/sklearn/feature_extraction/__init__.py b/sklearn/feature_extraction/__init__.py index 2103fc67589c3..4591bfc6980c8 100644 --- a/sklearn/feature_extraction/__init__.py +++ b/sklearn/feature_extraction/__init__.py @@ -5,7 +5,7 @@ """ from ._dict_vectorizer import DictVectorizer -from ._hashing import FeatureHasher +from ._hash import FeatureHasher from .image import img_to_graph, grid_to_graph from . import text diff --git a/sklearn/feature_extraction/_dict_vectorizer.py b/sklearn/feature_extraction/_dict_vectorizer.py index ca49263f57913..b527b0d72e6be 100644 --- a/sklearn/feature_extraction/_dict_vectorizer.py +++ b/sklearn/feature_extraction/_dict_vectorizer.py @@ -47,18 +47,17 @@ class DictVectorizer(TransformerMixin, BaseEstimator): Parameters ---------- - dtype : callable, optional + dtype : dtype, default=np.float64 The type of feature values. Passed to Numpy array/scipy.sparse matrix constructors as the dtype argument. - separator : string, optional + separator : str, default="=" Separator string used when constructing new features for one-hot coding. - sparse : boolean, optional. + sparse : bool, default=True Whether transform should produce scipy.sparse matrices. - True by default. - sort : boolean, optional. + sort : bool, default=True Whether ``feature_names_`` and ``vocabulary_`` should be - sorted when fitting. True by default. + sorted when fitting. Attributes ---------- @@ -241,13 +240,13 @@ def inverse_transform(self, X, dict_type=dict): ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) Sample matrix. - dict_type : callable, optional + dict_type : type, default=dict Constructor for feature mappings. Must conform to the collections.Mapping API. Returns ------- - D : list of dict_type objects, length = n_samples + D : list of dict_type objects of shape (n_samples,) Feature mappings for the samples in X. """ # COO matrix is not subscriptable @@ -276,7 +275,7 @@ def transform(self, X): Parameters ---------- - X : Mapping or iterable over Mappings, length = n_samples + X : Mapping or iterable over Mappings of shape (n_samples,) Dict(s) or Mapping(s) from feature names (arbitrary Python objects) to feature values (strings or convertible to dtype). @@ -324,7 +323,7 @@ def restrict(self, support, indices=False): support : array-like Boolean mask or list of indices (as returned by the get_support member of feature selectors). - indices : boolean, optional + indices : bool, default=False Whether support is a list of indices. Returns diff --git a/sklearn/feature_extraction/_hashing.py b/sklearn/feature_extraction/_hash.py similarity index 95% rename from sklearn/feature_extraction/_hashing.py rename to sklearn/feature_extraction/_hash.py index 83c3cf4857dbd..f52e6f296169b 100644 --- a/sklearn/feature_extraction/_hashing.py +++ b/sklearn/feature_extraction/_hash.py @@ -47,11 +47,11 @@ class FeatureHasher(TransformerMixin, BaseEstimator): Parameters ---------- - n_features : integer, optional + n_features : int, default=2**20 The number of features (columns) in the output matrices. Small numbers of features are likely to cause hash collisions, but large numbers will cause larger coefficient dimensions in linear learners. - input_type : string, optional, default "dict" + input_type : {"dict", "pair"}, default="dict" Either "dict" (the default) to accept dictionaries over (feature_name, value); "pair" to accept pairs of (feature_name, value); or "string" to accept single strings. @@ -60,11 +60,11 @@ class FeatureHasher(TransformerMixin, BaseEstimator): The feature_name is hashed to find the appropriate column for the feature. The value's sign might be flipped in the output (but see non_negative, below). - dtype : numpy type, optional, default np.float64 + dtype : numpy dtype, default=np.float64 The type of feature values. Passed to scipy.sparse matrix constructors as the dtype argument. Do not set this to bool, np.boolean or any unsigned integer type. - alternate_sign : boolean, optional, default True + alternate_sign : bool, default=True When True, an alternating sign is added to the features as to approximately conserve the inner product in the hashed space even for small n_features. This approach is similar to sparse random projection. @@ -101,7 +101,7 @@ def _validate_params(n_features, input_type): if not isinstance(n_features, numbers.Integral): raise TypeError("n_features must be integral, got %r (%s)." % (n_features, type(n_features))) - elif n_features < 1 or n_features >= 2 ** 31: + elif n_features < 1 or n_features >= np.iinfo(np.int32).max + 1: raise ValueError("Invalid number of features (%d)." % n_features) if input_type not in ("dict", "pair", "string"): @@ -116,7 +116,7 @@ def fit(self, X=None, y=None): Parameters ---------- - X : array-like + X : ndarray Returns ------- diff --git a/sklearn/feature_extraction/_hashing_fast.pyx b/sklearn/feature_extraction/_hashing_fast.pyx index 87980db0f435d..d5f8de592b5c6 100644 --- a/sklearn/feature_extraction/_hashing_fast.pyx +++ b/sklearn/feature_extraction/_hashing_fast.pyx @@ -88,7 +88,7 @@ def transform(raw_X, Py_ssize_t n_features, dtype, indices_a = np.frombuffer(indices, dtype=np.int32) indptr_a = np.frombuffer(indptr, dtype=indices_np_dtype) - if indptr[-1] > 2147483648: # = 2**31 + if indptr[-1] > np.iinfo(np.int32).max: # = 2**31 - 1 if sp_version < (0, 14): raise ValueError(('sparse CSR array has {} non-zero ' 'elements and requires 64 bit indexing, ' diff --git a/sklearn/feature_extraction/image.py b/sklearn/feature_extraction/image.py index 2cec6739e7f98..588abf3fcf896 100644 --- a/sklearn/feature_extraction/image.py +++ b/sklearn/feature_extraction/image.py @@ -33,11 +33,11 @@ def _make_edges_3d(n_x, n_y, n_z=1): Parameters ---------- - n_x : integer + n_x : int The size of the grid in the x direction. - n_y : integer + n_y : int The size of the grid in the y direction. - n_z : integer, optional + n_z : integer, default=1 The size of the grid in the z direction, defaults to 1 """ vertices = np.arange(n_x * n_y * n_z).reshape((n_x, n_y, n_z)) @@ -138,14 +138,16 @@ def img_to_graph(img, mask=None, return_as=sparse.coo_matrix, dtype=None): Parameters ---------- - img : ndarray, 2D or 3D - 2D or 3D image - mask : ndarray of booleans, optional + img : ndarray of shape (height, width) or (height, width, channel) + 2D or 3D image. + mask : ndarray of shape (height, width) or \ + (height, width, channel), dtype=bool, default=None An optional mask of the image, to consider only part of the pixels. - return_as : np.ndarray or a sparse matrix class, optional + return_as : np.ndarray or a sparse matrix class, \ + default=sparse.coo_matrix The class to use to build the returned adjacency matrix. - dtype : None or dtype, optional + dtype : dtype, default=None The data of the returned sparse matrix. By default it is the dtype of img @@ -175,14 +177,15 @@ def grid_to_graph(n_x, n_y, n_z=1, mask=None, return_as=sparse.coo_matrix, Dimension in x axis n_y : int Dimension in y axis - n_z : int, optional, default 1 + n_z : int, default=1 Dimension in z axis - mask : ndarray of booleans, optional + mask : ndarray of shape (n_x, n_y, n_z), dtype=bool, default=None An optional mask of the image, to consider only part of the pixels. - return_as : np.ndarray or a sparse matrix class, optional + return_as : np.ndarray or a sparse matrix class, \ + default=sparse.coo_matrix The class to use to build the returned adjacency matrix. - dtype : dtype, optional, default int + dtype : dtype, default=int The data of the returned sparse matrix. By default it is int Notes @@ -216,7 +219,7 @@ def _compute_n_patches(i_h, i_w, p_h, p_w, max_patches=None): The height of a patch p_w : int The width of a patch - max_patches : integer or float, optional default is None + max_patches : int or float, default=None The maximum number of patches to extract. If max_patches is a float between 0 and 1, it is taken to be a proportion of the total number of patches. @@ -257,12 +260,12 @@ def _extract_patches(arr, patch_shape=8, extraction_step=1): arr : ndarray n-dimensional array of which patches are to be extracted - patch_shape : integer or tuple of length arr.ndim + patch_shape : int or tuple of length arr.ndim.default=8 Indicates the shape of the patches to be extracted. If an integer is given, the shape will be a hypercube of sidelength given by its value. - extraction_step : integer or tuple of length arr.ndim + extraction_step : int or tuple of length arr.ndim, default=1 Indicates step size at which extraction shall be performed. If integer is given, then the step is uniform in all dimensions. @@ -298,6 +301,7 @@ def _extract_patches(arr, patch_shape=8, extraction_step=1): patches = as_strided(arr, shape=shape, strides=strides) return patches + @deprecated("The function feature_extraction.image.extract_patches has been " "deprecated in 0.22 and will be removed in 0.24.") def extract_patches(arr, patch_shape=8, extraction_step=1): @@ -316,12 +320,12 @@ def extract_patches(arr, patch_shape=8, extraction_step=1): arr : ndarray n-dimensional array of which patches are to be extracted - patch_shape : integer or tuple of length arr.ndim + patch_shape : int or tuple of length arr.ndim, default=8 Indicates the shape of the patches to be extracted. If an integer is given, the shape will be a hypercube of sidelength given by its value. - extraction_step : integer or tuple of length arr.ndim + extraction_step : int or tuple of length arr.ndim, default=1 Indicates step size at which extraction shall be performed. If integer is given, then the step is uniform in all dimensions. @@ -348,20 +352,20 @@ def extract_patches_2d(image, patch_size, max_patches=None, random_state=None): Parameters ---------- - image : array, shape = (image_height, image_width) or + image : ndarray of shape (image_height, image_width) or \ (image_height, image_width, n_channels) The original image data. For color images, the last dimension specifies the channel: a RGB image would have `n_channels=3`. - patch_size : tuple of ints (patch_height, patch_width) - the dimensions of one patch + patch_size : tuple of int (patch_height, patch_width) + The dimensions of one patch. - max_patches : integer or float, optional default is None - The maximum number of patches to extract. If max_patches is a float + max_patches : int or float, default=None + The maximum number of patches to extract. If `max_patches` is a float between 0 and 1, it is taken to be a proportion of the total number of patches. - random_state : int, RandomState instance or None, optional (default=None) + random_state : int, RandomState instance, default=None Determines the random number generator used for random sampling when `max_patches` is not None. Use an int to make the randomness deterministic. @@ -369,7 +373,7 @@ def extract_patches_2d(image, patch_size, max_patches=None, random_state=None): Returns ------- - patches : array, shape = (n_patches, patch_height, patch_width) or + patches : array of shape (n_patches, patch_height, patch_width) or \ (n_patches, patch_height, patch_width, n_channels) The collection of patches extracted from the image, where `n_patches` is either `max_patches` or the total number of patches that can be @@ -445,20 +449,20 @@ def reconstruct_from_patches_2d(patches, image_size): Parameters ---------- - patches : array, shape = (n_patches, patch_height, patch_width) or + patches : ndarray of shape (n_patches, patch_height, patch_width) or \ (n_patches, patch_height, patch_width, n_channels) The complete set of patches. If the patches contain colour information, channels are indexed along the last dimension: RGB patches would have `n_channels=3`. - image_size : tuple of ints (image_height, image_width) or + image_size : tuple of int (image_height, image_width) or \ (image_height, image_width, n_channels) - the size of the image that will be reconstructed + The size of the image that will be reconstructed. Returns ------- - image : array, shape = image_size - the reconstructed image + image : ndarray of shape image_size + The reconstructed image. """ i_h, i_w = image_size[:2] p_h, p_w = patches.shape[1:3] @@ -483,23 +487,24 @@ class PatchExtractor(BaseEstimator): Read more in the :ref:`User Guide `. + .. versionadded:: 0.9 + Parameters ---------- - patch_size : tuple of ints (patch_height, patch_width) - the dimensions of one patch + patch_size : tuple of int (patch_height, patch_width) + The dimensions of one patch. - max_patches : integer or float, optional default is None + max_patches : int or float, default=None The maximum number of patches per image to extract. If max_patches is a float in (0, 1), it is taken to mean a proportion of the total number of patches. - random_state : int, RandomState instance or None, optional (default=None) + random_state : int, RandomState instance, default=None Determines the random number generator used for random sampling when `max_patches` is not None. Use an int to make the randomness deterministic. See :term:`Glossary `. - Examples -------- >>> from sklearn.datasets import load_sample_images @@ -521,14 +526,14 @@ def __init__(self, patch_size=None, max_patches=None, random_state=None): self.random_state = random_state def fit(self, X, y=None): - """Do nothing and return the estimator unchanged + """Do nothing and return the estimator unchanged. This method is just there to implement the usual API and hence work in pipelines. Parameters ---------- - X : array-like, shape [n_samples, n_features] + X : array-like of shape (n_samples, n_features) Training data. """ return self @@ -538,7 +543,7 @@ def transform(self, X): Parameters ---------- - X : array, shape = (n_samples, image_height, image_width) or + X : ndarray of shape (n_samples, image_height, image_width) or \ (n_samples, image_height, image_width, n_channels) Array of images from which to extract patches. For color images, the last dimension specifies the channel: a RGB image would have @@ -546,7 +551,7 @@ def transform(self, X): Returns ------- - patches : array, shape = (n_patches, patch_height, patch_width) or + patches : array of shape (n_patches, patch_height, patch_width) or \ (n_patches, patch_height, patch_width, n_channels) The collection of patches extracted from the images, where `n_patches` is either `n_samples * max_patches` or the total diff --git a/sklearn/feature_extraction/tests/test_dict_vectorizer.py b/sklearn/feature_extraction/tests/test_dict_vectorizer.py index 7e7481a369646..22a7402908cf1 100644 --- a/sklearn/feature_extraction/tests/test_dict_vectorizer.py +++ b/sklearn/feature_extraction/tests/test_dict_vectorizer.py @@ -110,3 +110,12 @@ def test_deterministic_vocabulary(): v_2 = DictVectorizer().fit([d_shuffled]) assert v_1.vocabulary_ == v_2.vocabulary_ + + +def test_n_features_in(): + # For vectorizers, n_features_in_ does not make sense and does not exist. + dv = DictVectorizer() + assert not hasattr(dv, 'n_features_in_') + d = [{'foo': 1, 'bar': 2}, {'foo': 3, 'baz': 1}] + dv.fit(d) + assert not hasattr(dv, 'n_features_in_') diff --git a/sklearn/feature_extraction/tests/test_text.py b/sklearn/feature_extraction/tests/test_text.py index 25c353aae5276..86ae2fd6c149e 100644 --- a/sklearn/feature_extraction/tests/test_text.py +++ b/sklearn/feature_extraction/tests/test_text.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from collections.abc import Mapping import re -import warnings import pytest from scipy import sparse @@ -30,13 +29,12 @@ from numpy.testing import assert_array_almost_equal from numpy.testing import assert_array_equal from sklearn.utils import IS_PYPY -from sklearn.exceptions import ChangedBehaviorWarning from sklearn.utils._testing import (assert_almost_equal, - assert_warns_message, assert_raise_message, - clean_warning_registry, - SkipTest, assert_no_warnings, - fails_if_pypy, assert_allclose_dense_sparse, - skip_if_32bit) + assert_warns_message, assert_raise_message, + assert_no_warnings, + fails_if_pypy, + assert_allclose_dense_sparse, + skip_if_32bit) from collections import defaultdict from functools import partial import pickle @@ -298,18 +296,17 @@ def test_countvectorizer_custom_vocabulary_pipeline(): def test_countvectorizer_custom_vocabulary_repeated_indices(): vocab = {"pizza": 0, "beer": 0} - try: - CountVectorizer(vocabulary=vocab) - except ValueError as e: - assert "vocabulary contains repeated indices" in str(e).lower() + msg = "Vocabulary contains repeated indices" + with pytest.raises(ValueError, match=msg): + vect = CountVectorizer(vocabulary=vocab) + vect.fit(["pasta_siziliana"]) def test_countvectorizer_custom_vocabulary_gap_index(): vocab = {"pizza": 1, "beer": 2} - try: - CountVectorizer(vocabulary=vocab) - except ValueError as e: - assert "doesn't contain index" in str(e).lower() + with pytest.raises(ValueError, match="doesn't contain index"): + vect = CountVectorizer(vocabulary=vocab) + vect.fit(['pasta_verdura']) def test_countvectorizer_stop_words(): @@ -328,20 +325,14 @@ def test_countvectorizer_stop_words(): def test_countvectorizer_empty_vocabulary(): - try: + with pytest.raises(ValueError, match="empty vocabulary"): vect = CountVectorizer(vocabulary=[]) vect.fit(["foo"]) - assert False, "we shouldn't get here" - except ValueError as e: - assert "empty vocabulary" in str(e).lower() - try: + with pytest.raises(ValueError, match="empty vocabulary"): v = CountVectorizer(max_df=1.0, stop_words="english") # fit on stopwords only v.fit(["to be or not to be", "and me too", "and so do you"]) - assert False, "we shouldn't get here" - except ValueError as e: - assert "empty vocabulary" in str(e).lower() def test_fit_countvectorizer_twice(): @@ -389,16 +380,9 @@ def test_tfidf_no_smoothing(): [1, 0, 0]] tr = TfidfTransformer(smooth_idf=False, norm='l2') - clean_warning_registry() - with warnings.catch_warnings(record=True) as w: - 1. / np.array([0.]) - numpy_provides_div0_warning = len(w) == 1 - in_warning_message = 'divide by zero' - tfidf = assert_warns_message(RuntimeWarning, in_warning_message, - tr.fit_transform, X).toarray() - if not numpy_provides_div0_warning: - raise SkipTest("Numpy does not provide div 0 warnings.") + assert_warns_message(RuntimeWarning, in_warning_message, + tr.fit_transform, X).toarray() def test_sublinear_tf(): @@ -1099,6 +1083,7 @@ def test_vectorizer_string_object_as_input(Vectorizer): assert_raise_message( ValueError, message, vec.fit_transform, "hello world!") assert_raise_message(ValueError, message, vec.fit, "hello world!") + vec.fit(["some text", "some other text"]) assert_raise_message(ValueError, message, vec.transform, "hello world!") @@ -1157,7 +1142,7 @@ def test_vectorizers_invalid_ngram_range(vec): message = ("Invalid value for ngram_range=%s " "lower boundary larger than the upper boundary." % str(invalid_range)) - if isinstance(vec, HashingVectorizer): + if isinstance(vec, HashingVectorizer) and IS_PYPY: pytest.xfail(reason='HashingVectorizer is not supported on PyPy') assert_raise_message( @@ -1293,12 +1278,8 @@ def test_callable_analyzer_error(Estimator, input_type, err_type, err_msg): @pytest.mark.parametrize('input_type', ['file', 'filename']) def test_callable_analyzer_change_behavior(Estimator, analyzer, input_type): data = ['this is text, not file or filename'] - warn_msg = 'Since v0.21, vectorizer' with pytest.raises((FileNotFoundError, AttributeError)): - with pytest.warns(ChangedBehaviorWarning, match=warn_msg) as records: - Estimator(analyzer=analyzer, input=input_type).fit_transform(data) - assert len(records) == 1 - assert warn_msg in str(records[0]) + Estimator(analyzer=analyzer, input=input_type).fit_transform(data) @pytest.mark.parametrize( @@ -1361,6 +1342,18 @@ def test_unused_parameters_warn(Vectorizer, stop_words, vect.fit(train_data) +@pytest.mark.parametrize('Vectorizer, X', ( + (HashingVectorizer, [{'foo': 1, 'bar': 2}, {'foo': 3, 'baz': 1}]), + (CountVectorizer, JUNK_FOOD_DOCS)) +) +def test_n_features_in(Vectorizer, X): + # For vectorizers, n_features_in_ does not make sense + vectorizer = Vectorizer() + assert not hasattr(vectorizer, 'n_features_in_') + vectorizer.fit(X) + assert not hasattr(vectorizer, 'n_features_in_') + + # TODO: Remove in 0.24 def test_vectorizermixin_is_deprecated(): class MyVectorizer(VectorizerMixin): diff --git a/sklearn/feature_extraction/text.py b/sklearn/feature_extraction/text.py index dd293531184c3..4954329728d5e 100644 --- a/sklearn/feature_extraction/text.py +++ b/sklearn/feature_extraction/text.py @@ -27,12 +27,12 @@ from ..base import BaseEstimator, TransformerMixin from ..preprocessing import normalize -from ._hashing import FeatureHasher +from ._hash import FeatureHasher from ._stop_words import ENGLISH_STOP_WORDS from ..utils.validation import check_is_fitted, check_array, FLOAT_DTYPES from ..utils import _IS_32BIT, deprecated from ..utils.fixes import _astype_copy_false -from ..exceptions import ChangedBehaviorWarning, NotFittedError +from ..exceptions import NotFittedError __all__ = ['HashingVectorizer', @@ -53,10 +53,10 @@ def _preprocess(doc, accent_function=None, lower=False): ---------- doc: str The string to preprocess - accent_function: callable + accent_function: callable, default=None Function for handling accented characters. Common strategies include normalizing and removing. - lower: bool + lower: bool, default=False Whether to use str.lower to lowercase all fo the text Returns @@ -81,12 +81,12 @@ def _analyze(doc, analyzer=None, tokenizer=None, ngrams=None, Parameters ---------- - analyzer: callable - tokenizer: callable - ngrams: callable - preprocessor: callable - decoder: callable - stop_words: list + analyzer: callable, default=None + tokenizer: callable, default=None + ngrams: callable, default=None + preprocessor: callable, default=None + decoder: callable, default=None + stop_words: list, default=None Returns ------- @@ -123,7 +123,7 @@ def strip_accents_unicode(s): s : string The string to strip - See also + See Also -------- strip_accents_ascii Remove accentuated char for any unicode symbol that has a direct @@ -150,7 +150,7 @@ def strip_accents_ascii(s): s : string The string to strip - See also + See Also -------- strip_accents_unicode Remove accentuated char for any unicode symbol. @@ -190,14 +190,19 @@ class _VectorizerMixin: _white_spaces = re.compile(r"\s\s+") def decode(self, doc): - """Decode the input into a string of unicode symbols + """Decode the input into a string of unicode symbols. The decoding strategy depends on the vectorizer parameters. Parameters ---------- - doc : string - The string to decode + doc : str + The string to decode. + + Returns + ------- + doc: str + A string of unicode symbols. """ if self.input == 'filename': with open(doc, 'rb') as fh: @@ -298,7 +303,13 @@ def _char_wb_ngrams(self, text_document): return ngrams def build_preprocessor(self): - """Return a function to preprocess the text before tokenization""" + """Return a function to preprocess the text before tokenization. + + Returns + ------- + preprocessor: callable + A function to preprocess the text before tokenization. + """ if self.preprocessor is not None: return self.preprocessor @@ -320,14 +331,26 @@ def build_preprocessor(self): ) def build_tokenizer(self): - """Return a function that splits a string into a sequence of tokens""" + """Return a function that splits a string into a sequence of tokens. + + Returns + ------- + tokenizer: callable + A function to split a string into a sequence of tokens. + """ if self.tokenizer is not None: return self.tokenizer token_pattern = re.compile(self.token_pattern) return token_pattern.findall def get_stop_words(self): - """Build or fetch the effective stop words list""" + """Build or fetch the effective stop words list. + + Returns + ------- + stop_words: list or None + A list of stop words. + """ return _check_stop_list(self.stop_words) def _check_stop_words_consistency(self, stop_words, preprocess, tokenize): @@ -367,37 +390,18 @@ def _check_stop_words_consistency(self, stop_words, preprocess, tokenize): self._stop_words_id = id(self.stop_words) return 'error' - def _validate_custom_analyzer(self): - # This is to check if the given custom analyzer expects file or a - # filename instead of data. - # Behavior changed in v0.21, function could be removed in v0.23 - import tempfile - with tempfile.NamedTemporaryFile() as f: - fname = f.name - # now we're sure fname doesn't exist - - msg = ("Since v0.21, vectorizers pass the data to the custom analyzer " - "and not the file names or the file objects. This warning " - "will be removed in v0.23.") - try: - self.analyzer(fname) - except FileNotFoundError: - warnings.warn(msg, ChangedBehaviorWarning) - except AttributeError as e: - if str(e) == "'str' object has no attribute 'read'": - warnings.warn(msg, ChangedBehaviorWarning) - except Exception: - pass - def build_analyzer(self): """Return a callable that handles preprocessing, tokenization - and n-grams generation. + + Returns + ------- + analyzer: callable + A function to handle preprocessing, tokenization + and n-grams generation. """ if callable(self.analyzer): - if self.input in ['file', 'filename']: - self._validate_custom_analyzer() return partial( _analyze, analyzer=self.analyzer, decoder=self.decode ) @@ -548,7 +552,7 @@ class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator): Parameters ---------- - input : string {'filename', 'file', 'content'} + input : string {'filename', 'file', 'content'}, default='content' If 'filename', the sequence passed as an argument to fit is expected to be a list of filenames that need reading to fetch the raw content to analyze. @@ -563,13 +567,13 @@ class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator): If bytes or files are given to analyze, this encoding is used to decode. - decode_error : {'strict', 'ignore', 'replace'} + decode_error : {'strict', 'ignore', 'replace'}, default='strict' Instruction on what to do if a byte sequence is given to analyze that contains characters not of the given `encoding`. By default, it is 'strict', meaning that a UnicodeDecodeError will be raised. Other values are 'ignore' and 'replace'. - strip_accents : {'ascii', 'unicode', None} + strip_accents : {'ascii', 'unicode'}, default=None Remove accents and perform other character normalization during the preprocessing step. 'ascii' is a fast method that only works on characters that have @@ -580,20 +584,20 @@ class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator): Both 'ascii' and 'unicode' use NFKD normalization from :func:`unicodedata.normalize`. - lowercase : boolean, default=True + lowercase : bool, default=True Convert all characters to lowercase before tokenizing. - preprocessor : callable or None (default) + preprocessor : callable, default=None Override the preprocessing (string transformation) stage while preserving the tokenizing and n-grams generation steps. Only applies if ``analyzer is not callable``. - tokenizer : callable or None (default) + tokenizer : callable, default=None Override the string tokenization step while preserving the preprocessing and n-grams generation steps. Only applies if ``analyzer == 'word'``. - stop_words : string {'english'}, list, or None (default) + stop_words : string {'english'}, list, default=None If 'english', a built-in stop word list for English is used. There are several known issues with 'english' and you should consider an alternative (see :ref:`stop_words`). @@ -616,7 +620,8 @@ class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator): only bigrams. Only applies if ``analyzer is not callable``. - analyzer : string, {'word', 'char', 'char_wb'} or callable + analyzer : string, {'word', 'char', 'char_wb'} or callable, \ + default='word' Whether the feature should be made of word or character n-grams. Option 'char_wb' creates character n-grams only from text inside word boundaries; n-grams at the edges of words are padded with space. @@ -630,27 +635,27 @@ class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator): first read from the file and then passed to the given callable analyzer. - n_features : integer, default=(2 ** 20) + n_features : int, default=(2 ** 20) The number of features (columns) in the output matrices. Small numbers of features are likely to cause hash collisions, but large numbers will cause larger coefficient dimensions in linear learners. - binary : boolean, default=False. + binary : bool, default=False. If True, all non zero counts are set to 1. This is useful for discrete probabilistic models that model binary events rather than integer counts. - norm : 'l1', 'l2' or None, optional + norm : {'l1', 'l2'}, default='l2' Norm used to normalize term vectors. None for no normalization. - alternate_sign : boolean, optional, default True + alternate_sign : bool, default=True When True, an alternating sign is added to the features as to approximately conserve the inner product in the hashed space even for small n_features. This approach is similar to sparse random projection. .. versionadded:: 0.19 - dtype : type, optional + dtype : type, default=np.float64 Type of the matrix returned by fit_transform() or transform(). Examples @@ -667,11 +672,12 @@ class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator): >>> print(X.shape) (4, 16) - See also + See Also -------- CountVectorizer, TfidfVectorizer """ + def __init__(self, input='content', encoding='utf-8', decode_error='strict', strip_accents=None, lowercase=True, preprocessor=None, tokenizer=None, @@ -704,7 +710,7 @@ def partial_fit(self, X, y=None): Parameters ---------- - X : array-like, shape [n_samples, n_features] + X : ndarray of shape [n_samples, n_features] Training data. """ return self @@ -714,7 +720,7 @@ def fit(self, X, y=None): Parameters ---------- - X : array-like, shape [n_samples, n_features] + X : ndarray of shape [n_samples, n_features] Training data. """ # triggers a parameter validation @@ -810,7 +816,7 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator): Parameters ---------- - input : string {'filename', 'file', 'content'} + input : string {'filename', 'file', 'content'}, default='content' If 'filename', the sequence passed as an argument to fit is expected to be a list of filenames that need reading to fetch the raw content to analyze. @@ -821,17 +827,17 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator): Otherwise the input is expected to be a sequence of items that can be of type string or byte. - encoding : string, 'utf-8' by default. + encoding : string, default='utf-8' If bytes or files are given to analyze, this encoding is used to decode. - decode_error : {'strict', 'ignore', 'replace'} + decode_error : {'strict', 'ignore', 'replace'}, default='strict' Instruction on what to do if a byte sequence is given to analyze that contains characters not of the given `encoding`. By default, it is 'strict', meaning that a UnicodeDecodeError will be raised. Other values are 'ignore' and 'replace'. - strip_accents : {'ascii', 'unicode', None} + strip_accents : {'ascii', 'unicode'}, default=None Remove accents and perform other character normalization during the preprocessing step. 'ascii' is a fast method that only works on characters that have @@ -842,20 +848,20 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator): Both 'ascii' and 'unicode' use NFKD normalization from :func:`unicodedata.normalize`. - lowercase : boolean, True by default + lowercase : bool, default=True Convert all characters to lowercase before tokenizing. - preprocessor : callable or None (default) + preprocessor : callable, default=None Override the preprocessing (string transformation) stage while preserving the tokenizing and n-grams generation steps. Only applies if ``analyzer is not callable``. - tokenizer : callable or None (default) + tokenizer : callable, default=None Override the string tokenization step while preserving the preprocessing and n-grams generation steps. Only applies if ``analyzer == 'word'``. - stop_words : string {'english'}, list, or None (default) + stop_words : string {'english'}, list, default=None If 'english', a built-in stop word list for English is used. There are several known issues with 'english' and you should consider an alternative (see :ref:`stop_words`). @@ -876,14 +882,16 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator): ngram_range : tuple (min_n, max_n), default=(1, 1) The lower and upper boundary of the range of n-values for different - n-grams to be extracted. All values of n such that min_n <= n <= max_n - will be used. For example an ``ngram_range`` of ``(1, 1)`` means only - unigrams, ``(1, 2)`` means unigrams and bigrams, and ``(2, 2)`` means - only bigrams. + word n-grams or char n-grams to be extracted. All values of n such + such that min_n <= n <= max_n will be used. For example an + ``ngram_range`` of ``(1, 1)`` means only unigrams, ``(1, 2)`` means + unigrams and bigrams, and ``(2, 2)`` means only bigrams. Only applies if ``analyzer is not callable``. - analyzer : string, {'word', 'char', 'char_wb'} or callable - Whether the feature should be made of word or character n-grams. + analyzer : string, {'word', 'char', 'char_wb'} or callable, \ + default='word' + Whether the feature should be made of word n-gram or character + n-grams. Option 'char_wb' creates character n-grams only from text inside word boundaries; n-grams at the edges of words are padded with space. @@ -912,25 +920,25 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator): absolute counts. This parameter is ignored if vocabulary is not None. - max_features : int or None, default=None + max_features : int, default=None If not None, build a vocabulary that only consider the top max_features ordered by term frequency across the corpus. This parameter is ignored if vocabulary is not None. - vocabulary : Mapping or iterable, optional + vocabulary : Mapping or iterable, default=None Either a Mapping (e.g., a dict) where keys are terms and values are indices in the feature matrix, or an iterable over terms. If not given, a vocabulary is determined from the input documents. Indices in the mapping should not be repeated and should not have any gap between 0 and the largest index. - binary : boolean, default=False + binary : bool, default=False If True, all non zero counts are set to 1. This is useful for discrete probabilistic models that model binary events rather than integer counts. - dtype : type, optional + dtype : type, default=np.int64 Type of the matrix returned by fit_transform() or transform(). Attributes @@ -969,8 +977,19 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator): [0 2 0 1 0 1 1 0 1] [1 0 0 1 1 0 1 1 1] [0 1 1 1 0 0 1 0 1]] - - See also + >>> vectorizer2 = CountVectorizer(analyzer='word', ngram_range=(2, 2)) + >>> X2 = vectorizer2.fit_transform(corpus) + >>> print(vectorizer2.get_feature_names()) + ['and this', 'document is', 'first document', 'is the', 'is this', + 'second document', 'the first', 'the second', 'the third', 'third one', + 'this document', 'this is', 'this the'] + >>> print(X2.toarray()) + [[0 0 1 1 0 0 1 0 0 0 0 1 0] + [0 1 0 1 0 1 0 1 0 0 1 0 0] + [1 0 0 1 0 0 0 0 1 1 0 1 0] + [0 0 1 0 1 0 1 0 0 0 0 0 1]] + + See Also -------- HashingVectorizer, TfidfVectorizer @@ -1109,7 +1128,7 @@ def _count_vocab(self, raw_documents, fixed_vocab): raise ValueError("empty vocabulary; perhaps the documents only" " contain stop words") - if indptr[-1] > 2147483648: # = 2**31 - 1 + if indptr[-1] > np.iinfo(np.int32).max: # = 2**31 - 1 if _IS_32BIT: raise ValueError(('sparse CSR array has {} non-zero ' 'elements and requires 64 bit indexing, ' @@ -1158,7 +1177,7 @@ def fit_transform(self, raw_documents, y=None): Returns ------- - X : array, [n_samples, n_features] + X : array of shape (n_samples, n_features) Document-term matrix. """ # We intentionally don't call the transform method to make @@ -1182,8 +1201,6 @@ def fit_transform(self, raw_documents, y=None): X.data.fill(1) if not self.fixed_vocabulary_: - X = self._sort_features(X, vocabulary) - n_doc = X.shape[0] max_doc_count = (max_df if isinstance(max_df, numbers.Integral) @@ -1199,6 +1216,8 @@ def fit_transform(self, raw_documents, y=None): min_doc_count, max_features) + X = self._sort_features(X, vocabulary) + self.vocabulary_ = vocabulary return X @@ -1216,7 +1235,7 @@ def transform(self, raw_documents): Returns ------- - X : sparse matrix, [n_samples, n_features] + X : sparse matrix of shape (n_samples, n_features) Document-term matrix. """ if isinstance(raw_documents, str): @@ -1237,10 +1256,11 @@ def inverse_transform(self, X): Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) + Document-term matrix. Returns ------- - X_inv : list of arrays, len = n_samples + X_inv : list of arrays of shape (n_samples,) List of arrays of terms. """ self._check_vocabulary() @@ -1262,7 +1282,13 @@ def inverse_transform(self, X): for i in range(n_samples)] def get_feature_names(self): - """Array mapping from feature integer indices to feature name""" + """Array mapping from feature integer indices to feature name. + + Returns + ------- + feature_names : list + A list of feature names. + """ self._check_vocabulary() @@ -1323,7 +1349,7 @@ class TfidfTransformer(TransformerMixin, BaseEstimator): Parameters ---------- - norm : 'l1', 'l2' or None, optional (default='l2') + norm : {'l1', 'l2'}, default='l2' Each output row will have unit norm, either: * 'l2': Sum of squares of vector elements is 1. The cosine similarity between two vectors is their dot product when l2 norm has @@ -1331,20 +1357,20 @@ class TfidfTransformer(TransformerMixin, BaseEstimator): * 'l1': Sum of absolute values of vector elements is 1. See :func:`preprocessing.normalize` - use_idf : boolean (default=True) + use_idf : bool, default=True Enable inverse-document-frequency reweighting. - smooth_idf : boolean (default=True) + smooth_idf : bool, default=True Smooth idf weights by adding one to document frequencies, as if an extra document was seen containing every term in the collection exactly once. Prevents zero divisions. - sublinear_tf : boolean (default=False) + sublinear_tf : bool, default=False Apply sublinear tf scaling, i.e. replace tf with 1 + log(tf). Attributes ---------- - idf_ : array, shape (n_features) + idf_ : array of shape (n_features) The inverse document frequency (IDF) vector; only defined if ``use_idf`` is True. @@ -1392,12 +1418,12 @@ def __init__(self, norm='l2', use_idf=True, smooth_idf=True, self.sublinear_tf = sublinear_tf def fit(self, X, y=None): - """Learn the idf vector (global term weights) + """Learn the idf vector (global term weights). Parameters ---------- - X : sparse matrix, [n_samples, n_features] - a matrix of term/token counts + X : sparse matrix of shape n_samples, n_features) + A matrix of term/token counts. """ X = check_array(X, accept_sparse=('csr', 'csc')) if not sp.issparse(X): @@ -1428,16 +1454,16 @@ def transform(self, X, copy=True): Parameters ---------- - X : sparse matrix, [n_samples, n_features] + X : sparse matrix of (n_samples, n_features) a matrix of term/token counts - copy : boolean, default True + copy : bool, default=True Whether to copy X and operate on the copy or perform in-place operations. Returns ------- - vectors : sparse matrix, [n_samples, n_features] + vectors : sparse matrix of shape (n_samples, n_features) """ X = check_array(X, accept_sparse='csr', dtype=FLOAT_DTYPES, copy=copy) if not sp.issparse(X): @@ -1450,7 +1476,11 @@ def transform(self, X, copy=True): X.data += 1 if self.use_idf: - check_is_fitted(self, msg='idf vector is not fitted') + # idf_ being a property, the automatic attributes detection + # does not work as usual and we need to specify the attribute + # name: + check_is_fitted(self, attributes=["idf_"], + msg='idf vector is not fitted') expected_n_features = self._idf_diag.shape[0] if n_features != expected_n_features: @@ -1492,7 +1522,7 @@ class TfidfVectorizer(CountVectorizer): Parameters ---------- - input : string {'filename', 'file', 'content'} + input : {'filename', 'file', 'content'}, default='content' If 'filename', the sequence passed as an argument to fit is expected to be a list of filenames that need reading to fetch the raw content to analyze. @@ -1503,17 +1533,17 @@ class TfidfVectorizer(CountVectorizer): Otherwise the input is expected to be a sequence of items that can be of type string or byte. - encoding : string, 'utf-8' by default. + encoding : str, default='utf-8' If bytes or files are given to analyze, this encoding is used to decode. - decode_error : {'strict', 'ignore', 'replace'} (default='strict') + decode_error : {'strict', 'ignore', 'replace'}, default='strict' Instruction on what to do if a byte sequence is given to analyze that contains characters not of the given `encoding`. By default, it is 'strict', meaning that a UnicodeDecodeError will be raised. Other values are 'ignore' and 'replace'. - strip_accents : {'ascii', 'unicode', None} (default=None) + strip_accents : {'ascii', 'unicode'}, default=None Remove accents and perform other character normalization during the preprocessing step. 'ascii' is a fast method that only works on characters that have @@ -1524,20 +1554,20 @@ class TfidfVectorizer(CountVectorizer): Both 'ascii' and 'unicode' use NFKD normalization from :func:`unicodedata.normalize`. - lowercase : boolean (default=True) + lowercase : bool, default=True Convert all characters to lowercase before tokenizing. - preprocessor : callable or None (default=None) + preprocessor : callable, default=None Override the preprocessing (string transformation) stage while preserving the tokenizing and n-grams generation steps. Only applies if ``analyzer is not callable``. - tokenizer : callable or None (default=None) + tokenizer : callable, default=None Override the string tokenization step while preserving the preprocessing and n-grams generation steps. Only applies if ``analyzer == 'word'``. - analyzer : string, {'word', 'char', 'char_wb'} or callable + analyzer : {'word', 'char', 'char_wb'} or callable, default='word' Whether the feature should be made of word or character n-grams. Option 'char_wb' creates character n-grams only from text inside word boundaries; n-grams at the edges of words are padded with space. @@ -1551,7 +1581,7 @@ class TfidfVectorizer(CountVectorizer): first read from the file and then passed to the given callable analyzer. - stop_words : string {'english'}, list, or None (default=None) + stop_words : {'english'}, list, default=None If a string, it is passed to _check_stop_list and the appropriate stop list is returned. 'english' is currently the only supported string value. @@ -1566,7 +1596,7 @@ class TfidfVectorizer(CountVectorizer): in the range [0.7, 1.0) to automatically detect and filter stop words based on intra corpus document frequency of terms. - token_pattern : string + token_pattern : str Regular expression denoting what constitutes a "token", only used if ``analyzer == 'word'``. The default regexp selects tokens of 2 or more alphanumeric characters (punctuation is completely ignored @@ -1580,58 +1610,58 @@ class TfidfVectorizer(CountVectorizer): only bigrams. Only applies if ``analyzer is not callable``. - max_df : float in range [0.0, 1.0] or int (default=1.0) + max_df : float or int, default=1.0 When building the vocabulary ignore terms that have a document frequency strictly higher than the given threshold (corpus-specific stop words). - If float, the parameter represents a proportion of documents, integer - absolute counts. + If float in range [0.0, 1.0], the parameter represents a proportion of + documents, integer absolute counts. This parameter is ignored if vocabulary is not None. - min_df : float in range [0.0, 1.0] or int (default=1) + min_df : float or int, default=1 When building the vocabulary ignore terms that have a document frequency strictly lower than the given threshold. This value is also called cut-off in the literature. - If float, the parameter represents a proportion of documents, integer - absolute counts. + If float in range of [0.0, 1.0], the parameter represents a proportion + of documents, integer absolute counts. This parameter is ignored if vocabulary is not None. - max_features : int or None (default=None) + max_features : int, default=None If not None, build a vocabulary that only consider the top max_features ordered by term frequency across the corpus. This parameter is ignored if vocabulary is not None. - vocabulary : Mapping or iterable, optional (default=None) + vocabulary : Mapping or iterable, default=None Either a Mapping (e.g., a dict) where keys are terms and values are indices in the feature matrix, or an iterable over terms. If not given, a vocabulary is determined from the input documents. - binary : boolean (default=False) + binary : bool, default=False If True, all non-zero term counts are set to 1. This does not mean outputs will have only 0/1 values, only that the tf term in tf-idf - is binary. (Set idf and normalization to False to get 0/1 outputs.) + is binary. (Set idf and normalization to False to get 0/1 outputs). - dtype : type, optional (default=float64) + dtype : dtype, default=float64 Type of the matrix returned by fit_transform() or transform(). - norm : 'l1', 'l2' or None, optional (default='l2') + norm : {'l1', 'l2'}, default='l2' Each output row will have unit norm, either: * 'l2': Sum of squares of vector elements is 1. The cosine similarity between two vectors is their dot product when l2 norm has been applied. * 'l1': Sum of absolute values of vector elements is 1. - See :func:`preprocessing.normalize` + See :func:`preprocessing.normalize`. - use_idf : boolean (default=True) + use_idf : bool, default=True Enable inverse-document-frequency reweighting. - smooth_idf : boolean (default=True) + smooth_idf : bool, default=True Smooth idf weights by adding one to document frequencies, as if an extra document was seen containing every term in the collection exactly once. Prevents zero divisions. - sublinear_tf : boolean (default=False) + sublinear_tf : bool, default=False Apply sublinear tf scaling, i.e. replace tf with 1 + log(tf). Attributes @@ -1639,11 +1669,11 @@ class TfidfVectorizer(CountVectorizer): vocabulary_ : dict A mapping of terms to feature indices. - fixed_vocabulary_: boolean + fixed_vocabulary_: bool True if a fixed vocabulary of term to indices mapping is provided by the user - idf_ : array, shape (n_features) + idf_ : array of shape (n_features,) The inverse document frequency (IDF) vector; only defined if ``use_idf`` is True. @@ -1656,6 +1686,19 @@ class TfidfVectorizer(CountVectorizer): This is only available if no vocabulary was given. + See Also + -------- + CountVectorizer : Transforms text into a sparse matrix of n-gram counts. + + TfidfTransformer : Performs the TF-IDF transformation from a provided + matrix of counts. + + Notes + ----- + The ``stop_words_`` attribute can get large and increase the model size + when pickling. This attribute is provided only for introspection and can + be safely removed using delattr or set to None before pickling. + Examples -------- >>> from sklearn.feature_extraction.text import TfidfVectorizer @@ -1671,19 +1714,6 @@ class TfidfVectorizer(CountVectorizer): ['and', 'document', 'first', 'is', 'one', 'second', 'the', 'third', 'this'] >>> print(X.shape) (4, 9) - - See also - -------- - CountVectorizer : Transforms text into a sparse matrix of n-gram counts. - - TfidfTransformer : Performs the TF-IDF transformation from a provided - matrix of counts. - - Notes - ----- - The ``stop_words_`` attribute can get large and increase the model size - when pickling. This attribute is provided only for introspection and can - be safely removed using delattr or set to None before pickling. """ def __init__(self, input='content', encoding='utf-8', @@ -1770,11 +1800,14 @@ def fit(self, raw_documents, y=None): Parameters ---------- raw_documents : iterable - an iterable which yields either str, unicode or file objects + An iterable which yields either str, unicode or file objects. + y : None + This parameter is not needed to compute tfidf. Returns ------- - self : TfidfVectorizer + self : object + Fitted vectorizer. """ self._check_params() self._warn_for_unused_params() @@ -1791,11 +1824,13 @@ def fit_transform(self, raw_documents, y=None): Parameters ---------- raw_documents : iterable - an iterable which yields either str, unicode or file objects + An iterable which yields either str, unicode or file objects. + y : None + This parameter is ignored. Returns ------- - X : sparse matrix, [n_samples, n_features] + X : sparse matrix of (n_samples, n_features) Tf-idf-weighted document-term matrix. """ self._check_params() @@ -1814,9 +1849,9 @@ def transform(self, raw_documents, copy="deprecated"): Parameters ---------- raw_documents : iterable - an iterable which yields either str, unicode or file objects + An iterable which yields either str, unicode or file objects. - copy : boolean, default True + copy : bool, default=True Whether to copy X and operate on the copy or perform in-place operations. @@ -1827,10 +1862,10 @@ def transform(self, raw_documents, copy="deprecated"): Returns ------- - X : sparse matrix, [n_samples, n_features] + X : sparse matrix of (n_samples, n_features) Tf-idf-weighted document-term matrix. """ - check_is_fitted(self, msg='The tfidf vector is not fitted') + check_is_fitted(self, msg='The TF-IDF vectorizer is not fitted') # FIXME Remove copy parameter support in 0.24 if copy != "deprecated": diff --git a/sklearn/feature_selection/__init__.py b/sklearn/feature_selection/__init__.py index f8bda21a5813d..e9fa9ada1a5e4 100644 --- a/sklearn/feature_selection/__init__.py +++ b/sklearn/feature_selection/__init__.py @@ -24,6 +24,8 @@ from ._mutual_info import mutual_info_regression, mutual_info_classif +from ._base import SelectorMixin + __all__ = ['GenericUnivariateSelect', 'RFE', @@ -40,4 +42,5 @@ 'f_oneway', 'f_regression', 'mutual_info_classif', - 'mutual_info_regression'] + 'mutual_info_regression', + 'SelectorMixin'] diff --git a/sklearn/feature_selection/_from_model.py b/sklearn/feature_selection/_from_model.py index 674127f06acd7..dd72bddc58eb5 100644 --- a/sklearn/feature_selection/_from_model.py +++ b/sklearn/feature_selection/_from_model.py @@ -6,6 +6,7 @@ from ._base import SelectorMixin from ..base import BaseEstimator, clone, MetaEstimatorMixin +from ..utils.validation import check_is_fitted from ..exceptions import NotFittedError from ..utils.metaestimators import if_delegate_has_method @@ -254,6 +255,20 @@ def partial_fit(self, X, y=None, **fit_params): self.estimator_.partial_fit(X, y, **fit_params) return self + @property + def n_features_in_(self): + # For consistency with other estimators we raise a AttributeError so + # that hasattr() fails if the estimator isn't fitted. + try: + check_is_fitted(self) + except NotFittedError as nfe: + raise AttributeError( + "{} object has no n_features_in_ attribute." + .format(self.__class__.__name__) + ) from nfe + + return self.estimator_.n_features_in_ + def _more_tags(self): estimator_tags = self.estimator._get_tags() return {'allow_nan': estimator_tags.get('allow_nan', True)} diff --git a/sklearn/feature_selection/_mutual_info.py b/sklearn/feature_selection/_mutual_info.py index 95d1aeb183a27..5931e7bbc6ef5 100644 --- a/sklearn/feature_selection/_mutual_info.py +++ b/sklearn/feature_selection/_mutual_info.py @@ -224,12 +224,10 @@ def _estimate_mi(X, y, discrete_features='auto', discrete_target=False, data will be overwritten. random_state : int, RandomState instance or None, optional, default None - The seed of the pseudo random number generator for adding small noise - to continuous variables in order to remove repeated values. If int, - random_state is the seed used by the random number generator; If - RandomState instance, random_state is the random number generator; If - None, the random number generator is the RandomState instance used by - `np.random`. + Determines random number generation for adding small noise to + continuous variables in order to remove repeated values. + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary `. Returns ------- @@ -333,12 +331,10 @@ def mutual_info_regression(X, y, discrete_features='auto', n_neighbors=3, data will be overwritten. random_state : int, RandomState instance or None, optional, default None - The seed of the pseudo random number generator for adding small noise - to continuous variables in order to remove repeated values. - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + Determines random number generation for adding small noise to + continuous variables in order to remove repeated values. + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary `. Returns ------- @@ -412,12 +408,10 @@ def mutual_info_classif(X, y, discrete_features='auto', n_neighbors=3, data will be overwritten. random_state : int, RandomState instance or None, optional, default None - The seed of the pseudo random number generator for adding small noise - to continuous variables in order to remove repeated values. If int, - random_state is the seed used by the random number generator; If - RandomState instance, random_state is the random number generator; If - None, the random number generator is the RandomState instance used by - `np.random`. + Determines random number generation for adding small noise to + continuous variables in order to remove repeated values. + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary `. Returns ------- diff --git a/sklearn/feature_selection/_rfe.py b/sklearn/feature_selection/_rfe.py index 12e99175c9d61..69e3cc4de9e6c 100644 --- a/sklearn/feature_selection/_rfe.py +++ b/sklearn/feature_selection/_rfe.py @@ -155,8 +155,12 @@ def _fit(self, X, y, step_score=None): # self.scores_ will not be calculated when calling _fit through fit tags = self._get_tags() - X, y = check_X_y(X, y, "csc", ensure_min_features=2, - force_all_finite=not tags.get('allow_nan', True)) + X, y = self._validate_data( + X, y, accept_sparse="csc", + ensure_min_features=2, + force_all_finite=not tags.get('allow_nan', True), + multi_output=True + ) # Initialization n_features = X.shape[1] if self.n_features_to_select is None: @@ -489,8 +493,12 @@ def fit(self, X, y, groups=None): train/test set. Only used in conjunction with a "Group" :term:`cv` instance (e.g., :class:`~sklearn.model_selection.GroupKFold`). """ - X, y = check_X_y(X, y, "csr", ensure_min_features=2, - force_all_finite=False) + tags = self._get_tags() + X, y = self._validate_data( + X, y, accept_sparse="csr", ensure_min_features=2, + force_all_finite=not tags.get('allow_nan', True), + multi_output=True + ) # Initialization cv = check_cv(self.cv, y, is_classifier(self.estimator)) diff --git a/sklearn/feature_selection/_univariate_selection.py b/sklearn/feature_selection/_univariate_selection.py index 21990bb3a8167..221e46f2a505e 100644 --- a/sklearn/feature_selection/_univariate_selection.py +++ b/sklearn/feature_selection/_univariate_selection.py @@ -338,7 +338,8 @@ def fit(self, X, y): ------- self : object """ - X, y = check_X_y(X, y, ['csr', 'csc'], multi_output=True) + X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc'], + multi_output=True) if not callable(self.score_func): raise TypeError("The score function should be a callable, %s (%s) " diff --git a/sklearn/feature_selection/_variance_threshold.py b/sklearn/feature_selection/_variance_threshold.py index 4f9d720b762b9..6438e6b80dc0a 100644 --- a/sklearn/feature_selection/_variance_threshold.py +++ b/sklearn/feature_selection/_variance_threshold.py @@ -65,8 +65,9 @@ def fit(self, X, y=None): ------- self """ - X = check_array(X, ('csr', 'csc'), dtype=np.float64, - force_all_finite='allow-nan') + X = self._validate_data(X, accept_sparse=('csr', 'csc'), + dtype=np.float64, + force_all_finite='allow-nan') if hasattr(X, "toarray"): # sparse matrix _, self.variances_ = mean_variance_axis(X, axis=0) diff --git a/sklearn/feature_selection/tests/test_from_model.py b/sklearn/feature_selection/tests/test_from_model.py index 57bd88a30eb0e..89c1777b8c32c 100644 --- a/sklearn/feature_selection/tests/test_from_model.py +++ b/sklearn/feature_selection/tests/test_from_model.py @@ -37,8 +37,6 @@ def _more_tags(self): rng = np.random.RandomState(0) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_invalid_input(): clf = SGDClassifier(alpha=0.1, max_iter=10, shuffle=True, random_state=None, tol=None) @@ -252,8 +250,6 @@ def test_2d_coef(): assert_array_almost_equal(X_new, X[:, feature_mask]) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_partial_fit(): est = PassiveAggressiveClassifier(random_state=0, shuffle=False, max_iter=5, tol=None) @@ -284,8 +280,6 @@ def test_calling_fit_reinitializes(): assert transformer.estimator_.C == 100 -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_prefit(): # Test all possible combinations of the prefit parameter. @@ -325,8 +319,6 @@ def test_threshold_string(): assert_array_almost_equal(X_transform, data[:, mask]) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_threshold_without_refitting(): # Test that the threshold can be set without refitting the model. clf = SGDClassifier(alpha=0.1, max_iter=10, shuffle=True, diff --git a/sklearn/feature_selection/tests/test_rfe.py b/sklearn/feature_selection/tests/test_rfe.py index ccd3c0a1b0e83..ad0773edcb7b8 100644 --- a/sklearn/feature_selection/tests/test_rfe.py +++ b/sklearn/feature_selection/tests/test_rfe.py @@ -30,8 +30,8 @@ class MockClassifier: def __init__(self, foo_param=0): self.foo_param = foo_param - def fit(self, X, Y): - assert len(X) == len(Y) + def fit(self, X, y): + assert len(X) == len(y) self.coef_ = np.ones(X.shape[1], dtype=np.float64) return self @@ -42,12 +42,8 @@ def predict(self, T): decision_function = predict transform = predict - def score(self, X=None, Y=None): - if self.foo_param > 1: - score = 1. - else: - score = 0. - return score + def score(self, X=None, y=None): + return 0. def get_params(self, deep=True): return {'foo_param': self.foo_param} @@ -395,3 +391,15 @@ def test_rfe_allow_nan_inf_in_x(cv): rfe = RFE(estimator=clf) rfe.fit(X, y) rfe.transform(X) + + +@pytest.mark.parametrize('ClsRFE', [ + RFE, + RFECV + ]) +def test_multioutput(ClsRFE): + X = np.random.normal(size=(10, 3)) + y = np.random.randint(2, size=(10, 2)) + clf = RandomForestClassifier(n_estimators=5) + rfe_test = ClsRFE(clf) + rfe_test.fit(X, y) diff --git a/sklearn/feature_selection/tests/test_variance_threshold.py b/sklearn/feature_selection/tests/test_variance_threshold.py index 77d9c9445bc71..23e7703708984 100644 --- a/sklearn/feature_selection/tests/test_variance_threshold.py +++ b/sklearn/feature_selection/tests/test_variance_threshold.py @@ -11,6 +11,7 @@ [0, 2, 2, 3, 5], [1, 1, 2, 4, 0]] +data2 = [[-0.13725701]] * 10 def test_zero_variance(): # Test VarianceThreshold with default setting, zero variance. @@ -32,17 +33,16 @@ def test_variance_threshold(): assert (len(data), 1) == X.shape +@pytest.mark.skipif(np.var(data2) == 0, + reason=('This test is not valid for this platform, ' + 'as it relies on numerical instabilities.')) def test_zero_variance_floating_point_error(): # Test that VarianceThreshold(0.0).fit eliminates features that have # the same value in every sample, even when floating point errors # cause np.var not to be 0 for the feature. # See #13691 - data = [[-0.13725701]] * 10 - if np.var(data) == 0: - pytest.skip('This test is not valid for this platform, as it relies ' - 'on numerical instabilities.') - for X in [data, csr_matrix(data), csc_matrix(data), bsr_matrix(data)]: + for X in [data2, csr_matrix(data2), csc_matrix(data2), bsr_matrix(data2)]: msg = "No feature in X meets the variance threshold 0.00000" with pytest.raises(ValueError, match=msg): VarianceThreshold().fit(X) diff --git a/sklearn/gaussian_process/_gpc.py b/sklearn/gaussian_process/_gpc.py index 072cf80dba250..ed8ed2a007a22 100644 --- a/sklearn/gaussian_process/_gpc.py +++ b/sklearn/gaussian_process/_gpc.py @@ -49,12 +49,12 @@ class _BinaryGaussianProcessClassifierLaplace(BaseEstimator): Parameters ---------- - kernel : kernel object + kernel : kernel instance, default=None The kernel specifying the covariance function of the GP. If None is passed, the kernel "1.0 * RBF(1.0)" is used as default. Note that the kernel's hyperparameters are optimized during fitting. - optimizer : string or callable, optional (default: "fmin_l_bfgs_b") + optimizer : 'fmin_l_bfgs_b' or callable, default='fmin_l_bfgs_b' Can either be one of the internally supported optimizers for optimizing the kernel's parameters, specified by a string, or an externally defined optimizer passed as a callable. If a callable is passed, it @@ -79,7 +79,7 @@ def optimizer(obj_func, initial_theta, bounds): 'fmin_l_bfgs_b' - n_restarts_optimizer: int, optional (default: 0) + n_restarts_optimizer : int, default=0 The number of restarts of the optimizer for finding the kernel's parameters which maximize the log-marginal likelihood. The first run of the optimizer is performed from the kernel's initial parameters, @@ -88,12 +88,12 @@ def optimizer(obj_func, initial_theta, bounds): must be finite. Note that n_restarts_optimizer=0 implies that one run is performed. - max_iter_predict: int, optional (default: 100) + max_iter_predict : int, default=100 The maximum number of iterations in Newton's method for approximating the posterior during predict. Smaller values will reduce computation time at the cost of worse results. - warm_start : bool, optional (default: False) + warm_start : bool, default=False If warm-starts are enabled, the solution of the last Newton iteration on the Laplace approximation of the posterior mode is used as initialization for the next call of _posterior_mode(). This can speed @@ -101,22 +101,22 @@ def optimizer(obj_func, initial_theta, bounds): problems as in hyperparameter optimization. See :term:`the Glossary `. - copy_X_train : bool, optional (default: True) + copy_X_train : bool, default=True If True, a persistent copy of the training data is stored in the object. Otherwise, just a reference to the training data is stored, which might cause predictions to change if the data is modified externally. - random_state : int, RandomState instance or None, optional (default: None) - The generator used to initialize the centers. If int, random_state is - the seed used by the random number generator; If RandomState instance, - random_state is the random number generator; If None, the random number - generator is the RandomState instance used by `np.random`. + random_state : int or RandomState, default=None + Determines random number generation used to initialize the centers. + Pass an int for reproducible results across multiple function calls. + See :term: `Glossary `. Attributes ---------- - X_train_ : array-like of shape (n_samples, n_features) - Feature values in training data (also required for prediction) + X_train_ : array-like of shape (n_samples, n_features) or list of object + Feature vectors or other representations of training data (also + required for prediction). y_train_ : array-like of shape (n_samples,) Target values in training data (also required for prediction) @@ -124,7 +124,7 @@ def optimizer(obj_func, initial_theta, bounds): classes_ : array-like of shape (n_classes,) Unique class labels. - kernel_ : kernel object + kernel_ : kernl instance The kernel used for prediction. The structure of the kernel is the same as the one passed as parameter but with optimized hyperparameters @@ -160,8 +160,8 @@ def fit(self, X, y): Parameters ---------- - X : array-like of shape (n_samples, n_features) - Training data + X : array-like of shape (n_samples, n_features) or list of object + Feature vectors or other representations of training data. y : array-like of shape (n_samples,) Target values, must be binary @@ -248,7 +248,8 @@ def predict(self, X): Parameters ---------- - X : array-like of shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) or list of object + Query points where the GP is evaluated for classification. Returns ------- @@ -270,7 +271,8 @@ def predict_proba(self, X): Parameters ---------- - X : array-like of shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) or list of object + Query points where the GP is evaluated for classification. Returns ------- @@ -310,12 +312,12 @@ def log_marginal_likelihood(self, theta=None, eval_gradient=False, Parameters ---------- - theta : array-like of shape (n_kernel_params,) or None + theta : array-like of shape (n_kernel_params,), default=None Kernel hyperparameters for which the log-marginal likelihood is evaluated. If None, the precomputed log_marginal_likelihood of ``self.kernel_.theta`` is returned. - eval_gradient : bool, default: False + eval_gradient : bool, default=False If True, the gradient of the log-marginal likelihood with respect to the kernel hyperparameters at position theta is returned additionally. If True, theta must not be None. @@ -329,10 +331,11 @@ def log_marginal_likelihood(self, theta=None, eval_gradient=False, log_likelihood : float Log-marginal likelihood of theta for training data. - log_likelihood_gradient : array, shape = (n_kernel_params,), optional + log_likelihood_gradient : ndarray of shape (n_kernel_params,), \ + optional Gradient of the log-marginal likelihood with respect to the kernel hyperparameters at position theta. - Only returned when eval_gradient is True. + Only returned when `eval_gradient` is True. """ if theta is None: if eval_gradient: @@ -464,14 +467,16 @@ class GaussianProcessClassifier(ClassifierMixin, BaseEstimator): classifiers are fitted. Note that this class thus does not implement a true multi-class Laplace approximation. + Read more in the :ref:`User Guide `. + Parameters ---------- - kernel : kernel object + kernel : kernel instance, default=None The kernel specifying the covariance function of the GP. If None is passed, the kernel "1.0 * RBF(1.0)" is used as default. Note that the kernel's hyperparameters are optimized during fitting. - optimizer : string or callable, optional (default: "fmin_l_bfgs_b") + optimizer : 'fmin_l_bfgs_b' or callable, default='fmin_l_bfgs_b' Can either be one of the internally supported optimizers for optimizing the kernel's parameters, specified by a string, or an externally defined optimizer passed as a callable. If a callable is passed, it @@ -496,7 +501,7 @@ def optimizer(obj_func, initial_theta, bounds): 'fmin_l_bfgs_b' - n_restarts_optimizer : int, optional (default: 0) + n_restarts_optimizer : int, default=0 The number of restarts of the optimizer for finding the kernel's parameters which maximize the log-marginal likelihood. The first run of the optimizer is performed from the kernel's initial parameters, @@ -505,12 +510,12 @@ def optimizer(obj_func, initial_theta, bounds): must be finite. Note that n_restarts_optimizer=0 implies that one run is performed. - max_iter_predict : int, optional (default: 100) + max_iter_predict : int, default=100 The maximum number of iterations in Newton's method for approximating the posterior during predict. Smaller values will reduce computation time at the cost of worse results. - warm_start : bool, optional (default: False) + warm_start : bool, default=False If warm-starts are enabled, the solution of the last Newton iteration on the Laplace approximation of the posterior mode is used as initialization for the next call of _posterior_mode(). This can speed @@ -518,31 +523,29 @@ def optimizer(obj_func, initial_theta, bounds): problems as in hyperparameter optimization. See :term:`the Glossary `. - copy_X_train : bool, optional (default: True) + copy_X_train : bool, default=True If True, a persistent copy of the training data is stored in the object. Otherwise, just a reference to the training data is stored, which might cause predictions to change if the data is modified externally. - random_state : int, RandomState instance or None, optional (default: None) - The generator used to initialize the centers. - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState, default=None + Determines random number generation used to initialize the centers. + Pass an int for reproducible results across multiple function calls. + See :term: `Glossary `. - multi_class : string, default : "one_vs_rest" + multi_class : {'one_vs_rest', 'one_vs_one'}, default='one_vs_rest' Specifies how multi-class classification problems are handled. - Supported are "one_vs_rest" and "one_vs_one". In "one_vs_rest", + Supported are 'one_vs_rest' and 'one_vs_one'. In 'one_vs_rest', one binary Gaussian process classifier is fitted for each class, which - is trained to separate this class from the rest. In "one_vs_one", one + is trained to separate this class from the rest. In 'one_vs_one', one binary Gaussian process classifier is fitted for each pair of classes, which is trained to separate these two classes. The predictions of these binary predictors are combined into multi-class predictions. - Note that "one_vs_one" does not support predicting probability + Note that 'one_vs_one' does not support predicting probability estimates. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of jobs to use for the computation. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` @@ -550,7 +553,7 @@ def optimizer(obj_func, initial_theta, bounds): Attributes ---------- - kernel_ : kernel object + kernel_ : kernel instance The kernel used for prediction. In case of binary classification, the structure of the kernel is the same as the one passed as parameter but with optimized hyperparameters. In case of multi-class @@ -602,8 +605,8 @@ def fit(self, X, y): Parameters ---------- - X : array-like of shape (n_samples, n_features) - Training data + X : array-like of shape (n_samples, n_features) or list of object + Feature vectors or other representations of training data. y : array-like of shape (n_samples,) Target values, must be binary @@ -612,7 +615,12 @@ def fit(self, X, y): ------- self : returns an instance of self. """ - X, y = check_X_y(X, y, multi_output=False) + if self.kernel is None or self.kernel.requires_vector_input: + X, y = self._validate_data(X, y, multi_output=False, + ensure_2d=True, dtype="numeric") + else: + X, y = self._validate_data(X, y, multi_output=False, + ensure_2d=False, dtype=None) self.base_estimator_ = _BinaryGaussianProcessClassifierLaplace( self.kernel, self.optimizer, self.n_restarts_optimizer, @@ -656,7 +664,8 @@ def predict(self, X): Parameters ---------- - X : array-like of shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) or list of object + Query points where the GP is evaluated for classification. Returns ------- @@ -664,7 +673,12 @@ def predict(self, X): Predicted target values for X, values are from ``classes_`` """ check_is_fitted(self) - X = check_array(X) + + if self.kernel is None or self.kernel.requires_vector_input: + X = check_array(X, ensure_2d=True, dtype="numeric") + else: + X = check_array(X, ensure_2d=False, dtype=None) + return self.base_estimator_.predict(X) def predict_proba(self, X): @@ -672,7 +686,8 @@ def predict_proba(self, X): Parameters ---------- - X : array-like of shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) or list of object + Query points where the GP is evaluated for classification. Returns ------- @@ -686,7 +701,12 @@ def predict_proba(self, X): raise ValueError("one_vs_one multi-class mode does not support " "predicting probability estimates. Use " "one_vs_rest mode instead.") - X = check_array(X) + + if self.kernel is None or self.kernel.requires_vector_input: + X = check_array(X, ensure_2d=True, dtype="numeric") + else: + X = check_array(X, ensure_2d=False, dtype=None) + return self.base_estimator_.predict_proba(X) @property @@ -707,7 +727,7 @@ def log_marginal_likelihood(self, theta=None, eval_gradient=False, Parameters ---------- - theta : array-like of shape (n_kernel_params,) or None + theta : array-like of shape (n_kernel_params,), default=None Kernel hyperparameters for which the log-marginal likelihood is evaluated. In the case of multi-class classification, theta may be the hyperparameters of the compound kernel or of an individual @@ -715,7 +735,7 @@ def log_marginal_likelihood(self, theta=None, eval_gradient=False, same theta values. If None, the precomputed log_marginal_likelihood of ``self.kernel_.theta`` is returned. - eval_gradient : bool, default: False + eval_gradient : bool, default=False If True, the gradient of the log-marginal likelihood with respect to the kernel hyperparameters at position theta is returned additionally. Note that gradient computation is not supported @@ -730,10 +750,10 @@ def log_marginal_likelihood(self, theta=None, eval_gradient=False, log_likelihood : float Log-marginal likelihood of theta for training data. - log_likelihood_gradient : array, shape = (n_kernel_params,), optional + log_likelihood_gradient : ndarray of shape (n_kernel_params,), optional Gradient of the log-marginal likelihood with respect to the kernel hyperparameters at position theta. - Only returned when eval_gradient is True. + Only returned when `eval_gradient` is True. """ check_is_fitted(self) diff --git a/sklearn/gaussian_process/_gpr.py b/sklearn/gaussian_process/_gpr.py index a2be69abff794..1b48efb39f26d 100644 --- a/sklearn/gaussian_process/_gpr.py +++ b/sklearn/gaussian_process/_gpr.py @@ -1,7 +1,7 @@ """Gaussian processes regression. """ # Authors: Jan Hendrik Metzen -# +# Modified by: Pete Green # License: BSD 3 clause import warnings @@ -42,12 +42,12 @@ class GaussianProcessRegressor(MultiOutputMixin, Parameters ---------- - kernel : kernel object + kernel : kernel instance, default=None The kernel specifying the covariance function of the GP. If None is passed, the kernel "1.0 * RBF(1.0)" is used as default. Note that the kernel's hyperparameters are optimized during fitting. - alpha : float or array-like, optional (default: 1e-10) + alpha : float or array-like of shape (n_samples), default=1e-10 Value added to the diagonal of the kernel matrix during fitting. Larger values correspond to increased noise level in the observations. This can also prevent a potential numerical issue during fitting, by @@ -58,7 +58,7 @@ class GaussianProcessRegressor(MultiOutputMixin, Allowing to specify the noise level directly as a parameter is mainly for convenience and for consistency with Ridge. - optimizer : string or callable, optional (default: "fmin_l_bfgs_b") + optimizer : "fmin_l_bfgs_b" or callable, default="fmin_l_bfgs_b" Can either be one of the internally supported optimizers for optimizing the kernel's parameters, specified by a string, or an externally defined optimizer passed as a callable. If a callable is passed, it @@ -83,7 +83,7 @@ def optimizer(obj_func, initial_theta, bounds): 'fmin_l_bfgs_b' - n_restarts_optimizer : int, optional (default: 0) + n_restarts_optimizer : int, default=0 The number of restarts of the optimizer for finding the kernel's parameters which maximize the log-marginal likelihood. The first run of the optimizer is performed from the kernel's initial parameters, @@ -93,34 +93,35 @@ def optimizer(obj_func, initial_theta, bounds): run is performed. normalize_y : boolean, optional (default: False) - Whether the target values y are normalized, i.e., the mean of the - observed target values become zero. This parameter should be set to - True if the target values' mean is expected to differ considerable from - zero. When enabled, the normalization effectively modifies the GP's - prior based on the data, which contradicts the likelihood principle; - normalization is thus disabled per default. - - copy_X_train : bool, optional (default: True) + Whether the target values y are normalized, the mean and variance of + the target values are set equal to 0 and 1 respectively. This is + recommended for cases where zero-mean, unit-variance priors are used. + Note that, in this implementation, the normalisation is reversed + before the GP predictions are reported. + + .. versionchanged:: 0.23 + + copy_X_train : bool, default=True If True, a persistent copy of the training data is stored in the object. Otherwise, just a reference to the training data is stored, which might cause predictions to change if the data is modified externally. - random_state : int, RandomState instance or None, optional (default: None) - The generator used to initialize the centers. If int, random_state is - the seed used by the random number generator; If RandomState instance, - random_state is the random number generator; If None, the random number - generator is the RandomState instance used by `np.random`. + random_state : int or RandomState, default=None + Determines random number generation used to initialize the centers. + Pass an int for reproducible results across multiple function calls. + See :term: `Glossary `. Attributes ---------- - X_train_ : array-like of shape (n_samples, n_features) - Feature values in training data (also required for prediction) + X_train_ : array-like of shape (n_samples, n_features) or list of object + Feature vectors or other representations of training data (also + required for prediction). y_train_ : array-like of shape (n_samples,) or (n_samples, n_targets) Target values in training data (also required for prediction) - kernel_ : kernel object + kernel_ : kernel instance The kernel used for prediction. The structure of the kernel is the same as the one passed as parameter but with optimized hyperparameters @@ -164,8 +165,8 @@ def fit(self, X, y): Parameters ---------- - X : array-like of shape (n_samples, n_features) - Training data + X : array-like of shape (n_samples, n_features) or list of object + Feature vectors or other representations of training data. y : array-like of shape (n_samples,) or (n_samples, n_targets) Target values @@ -182,15 +183,24 @@ def fit(self, X, y): self._rng = check_random_state(self.random_state) - X, y = check_X_y(X, y, multi_output=True, y_numeric=True) + if self.kernel_.requires_vector_input: + X, y = self._validate_data(X, y, multi_output=True, y_numeric=True, + ensure_2d=True, dtype="numeric") + else: + X, y = self._validate_data(X, y, multi_output=True, y_numeric=True, + ensure_2d=False, dtype=None) # Normalize target value if self.normalize_y: self._y_train_mean = np.mean(y, axis=0) - # demean y - y = y - self._y_train_mean + self._y_train_std = np.std(y, axis=0) + + # Remove mean and make unit variance + y = (y - self._y_train_mean) / self._y_train_std + else: self._y_train_mean = np.zeros(1) + self._y_train_std = 1 if np.iterable(self.alpha) \ and self.alpha.shape[0] != y.shape[0]: @@ -273,36 +283,39 @@ def predict(self, X, return_std=False, return_cov=False): Parameters ---------- - X : array-like of shape (n_samples, n_features) - Query points where the GP is evaluated + X : array-like of shape (n_samples, n_features) or list of object + Query points where the GP is evaluated. - return_std : bool, default: False + return_std : bool, default=False If True, the standard-deviation of the predictive distribution at the query points is returned along with the mean. - return_cov : bool, default: False + return_cov : bool, default=False If True, the covariance of the joint predictive distribution at the query points is returned along with the mean Returns ------- - y_mean : array, shape = (n_samples, [n_output_dims]) + y_mean : ndarray of shape (n_samples, [n_output_dims]) Mean of predictive distribution a query points - y_std : array, shape = (n_samples,), optional + y_std : ndarray of shape (n_samples,), optional Standard deviation of predictive distribution at query points. - Only returned when return_std is True. + Only returned when `return_std` is True. - y_cov : array, shape = (n_samples, n_samples), optional + y_cov : ndarray of shape (n_samples, n_samples), optional Covariance of joint predictive distribution a query points. - Only returned when return_cov is True. + Only returned when `return_cov` is True. """ if return_std and return_cov: raise RuntimeError( "Not returning standard deviation of predictions when " "returning full covariance.") - X = check_array(X) + if self.kernel is None or self.kernel.requires_vector_input: + X = check_array(X, ensure_2d=True, dtype="numeric") + else: + X = check_array(X, ensure_2d=False, dtype=None) if not hasattr(self, "X_train_"): # Unfitted;predict based on GP prior if self.kernel is None: @@ -322,10 +335,17 @@ def predict(self, X, return_std=False, return_cov=False): else: # Predict based on GP posterior K_trans = self.kernel_(X, self.X_train_) y_mean = K_trans.dot(self.alpha_) # Line 4 (y_mean = f_star) - y_mean = self._y_train_mean + y_mean # undo normal. + + # undo normalisation + y_mean = self._y_train_std * y_mean + self._y_train_mean + if return_cov: v = cho_solve((self.L_, True), K_trans.T) # Line 5 y_cov = self.kernel_(X) - K_trans.dot(v) # Line 6 + + # undo normalisation + y_cov = y_cov * self._y_train_std**2 + return y_mean, y_cov elif return_std: # cache result of K_inv computation @@ -348,6 +368,10 @@ def predict(self, X, return_std=False, return_cov=False): warnings.warn("Predicted variances smaller than 0. " "Setting those variances to 0.") y_var[y_var_negative] = 0.0 + + # undo normalisation + y_var = y_var * self._y_train_std**2 + return y_mean, np.sqrt(y_var) else: return y_mean @@ -357,21 +381,21 @@ def sample_y(self, X, n_samples=1, random_state=0): Parameters ---------- - X : array-like of shape (n_samples_X, n_features) - Query points where the GP samples are evaluated + X : array-like of shape (n_samples, n_features) or list of object + Query points where the GP is evaluated. - n_samples : int, default: 1 + n_samples : int, default=1 The number of samples drawn from the Gaussian process - random_state : int, RandomState instance or None, optional (default=0) - If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the - random number generator; If None, the random number - generator is the RandomState instance used by `np.random`. + random_state : int, RandomState, default=0 + Determines random number generation to randomly draw samples. + Pass an int for reproducible results across multiple function + calls. + See :term: `Glossary `. Returns ------- - y_samples : array, shape = (n_samples_X, [n_output_dims], n_samples) + y_samples : ndarray of shape (n_samples_X, [n_output_dims], n_samples) Values of n_samples samples drawn from Gaussian process and evaluated at query points. """ @@ -394,12 +418,12 @@ def log_marginal_likelihood(self, theta=None, eval_gradient=False, Parameters ---------- - theta : array-like of shape (n_kernel_params,) or None + theta : array-like of shape (n_kernel_params,) default=None Kernel hyperparameters for which the log-marginal likelihood is evaluated. If None, the precomputed log_marginal_likelihood of ``self.kernel_.theta`` is returned. - eval_gradient : bool, default: False + eval_gradient : bool, default=False If True, the gradient of the log-marginal likelihood with respect to the kernel hyperparameters at position theta is returned additionally. If True, theta must not be None. @@ -413,7 +437,7 @@ def log_marginal_likelihood(self, theta=None, eval_gradient=False, log_likelihood : float Log-marginal likelihood of theta for training data. - log_likelihood_gradient : array, shape = (n_kernel_params,), optional + log_likelihood_gradient : ndarray of shape (n_kernel_params,), optional Gradient of the log-marginal likelihood with respect to the kernel hyperparameters at position theta. Only returned when eval_gradient is True. diff --git a/sklearn/gaussian_process/kernels.py b/sklearn/gaussian_process/kernels.py index 1634113a009f3..bf48aac36d846 100644 --- a/sklearn/gaussian_process/kernels.py +++ b/sklearn/gaussian_process/kernels.py @@ -31,6 +31,7 @@ from ..metrics.pairwise import pairwise_kernels from ..base import clone +from ..utils.validation import _num_samples def _check_length_scale(X, length_scale): @@ -53,12 +54,12 @@ class Hyperparameter(namedtuple('Hyperparameter', Attributes ---------- - name : string + name : str The name of the hyperparameter. Note that a kernel using a hyperparameter with name "x" must have the attributes self.x and self.x_bounds - value_type : string + value_type : str The type of the hyperparameter. Currently, only "numeric" hyperparameters are supported. @@ -74,7 +75,7 @@ class Hyperparameter(namedtuple('Hyperparameter', corresponds to a hyperparameter which is vector-valued, such as, e.g., anisotropic length-scales. - fixed : bool, default: None + fixed : bool, default=None Whether the value of this hyperparameter is fixed, i.e., cannot be changed during hyperparameter tuning. If None is passed, the "fixed" is derived based on the given bounds. @@ -127,13 +128,13 @@ def get_params(self, deep=True): Parameters ---------- - deep : boolean, optional + deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns ------- - params : mapping of string to any + params : dict Parameter names mapped to their values. """ params = dict() @@ -212,7 +213,7 @@ def clone_with_theta(self, theta): Parameters ---------- - theta : array, shape (n_dims,) + theta : ndarray of shape (n_dims,) The hyperparameters """ cloned = clone(self) @@ -242,7 +243,7 @@ def theta(self): Returns ------- - theta : array, shape (n_dims,) + theta : ndarray of shape (n_dims,) The non-fixed, log-transformed hyperparameters of the kernel """ theta = [] @@ -261,7 +262,7 @@ def theta(self, theta): Parameters ---------- - theta : array, shape (n_dims,) + theta : ndarray of shape (n_dims,) The non-fixed, log-transformed hyperparameters of the kernel """ params = self.get_params() @@ -290,7 +291,7 @@ def bounds(self): Returns ------- - bounds : array, shape (n_dims, 2) + bounds : ndarray of shape (n_dims, 2) The log-transformed bounds on the kernel's hyperparameters theta """ bounds = [hyperparameter.bounds @@ -352,12 +353,12 @@ def diag(self, X): Parameters ---------- - X : array, shape (n_samples_X, n_features) + X : array-like of shape (n_samples,) Left argument of the returned kernel k(X, Y) Returns ------- - K_diag : array, shape (n_samples_X,) + K_diag : ndarray of shape (n_samples_X,) Diagonal of kernel k(X, X) """ @@ -365,6 +366,13 @@ def diag(self, X): def is_stationary(self): """Returns whether the kernel is stationary. """ + @property + def requires_vector_input(self): + """Returns whether the kernel is defined on fixed-length feature + vectors or generic objects. Defaults to True for backward + compatibility.""" + return True + class NormalizedKernelMixin: """Mixin for kernels which are normalized: k(X, X)=1. @@ -381,12 +389,12 @@ def diag(self, X): Parameters ---------- - X : array, shape (n_samples_X, n_features) + X : ndarray of shape (n_samples_X, n_features) Left argument of the returned kernel k(X, Y) Returns ------- - K_diag : array, shape (n_samples_X,) + K_diag : ndarray of shape (n_samples_X,) Diagonal of kernel k(X, X) """ return np.ones(X.shape[0]) @@ -403,6 +411,19 @@ def is_stationary(self): return True +class GenericKernelMixin: + """Mixin for kernels which operate on generic objects such as variable- + length sequences, trees, and graphs. + + .. versionadded:: 0.22 + """ + + @property + def requires_vector_input(self): + """Whether the kernel works only on fixed-length feature vectors.""" + return False + + class CompoundKernel(Kernel): """Kernel which is composed of a set of other kernels. @@ -410,7 +431,7 @@ class CompoundKernel(Kernel): Parameters ---------- - kernels : list of Kernel objects + kernels : list of Kernels The other kernels """ @@ -422,13 +443,13 @@ def get_params(self, deep=True): Parameters ---------- - deep : boolean, optional + deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns ------- - params : mapping of string to any + params : dict Parameter names mapped to their values. """ return dict(kernels=self.kernels) @@ -444,7 +465,7 @@ def theta(self): Returns ------- - theta : array, shape (n_dims,) + theta : ndarray of shape (n_dims,) The non-fixed, log-transformed hyperparameters of the kernel """ return np.hstack([kernel.theta for kernel in self.kernels]) @@ -455,7 +476,7 @@ def theta(self, theta): Parameters ---------- - theta : array, shape (n_dims,) + theta : array of shape (n_dims,) The non-fixed, log-transformed hyperparameters of the kernel """ k_dims = self.k1.n_dims @@ -468,7 +489,7 @@ def bounds(self): Returns ------- - bounds : array, shape (n_dims, 2) + bounds : array of shape (n_dims, 2) The log-transformed bounds on the kernel's hyperparameters theta """ return np.vstack([kernel.bounds for kernel in self.kernels]) @@ -481,25 +502,28 @@ def __call__(self, X, Y=None, eval_gradient=False): Parameters ---------- - X : array, shape (n_samples_X, n_features) + X : array-like of shape (n_samples_X, n_features) or list of object, \ + default=None Left argument of the returned kernel k(X, Y) - Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Y : array-like of shape (n_samples_X, n_features) or list of object, \ + default=None Right argument of the returned kernel k(X, Y). If None, k(X, X) - if evaluated instead. + is evaluated instead. - eval_gradient : bool (optional, default=False) + eval_gradient : bool, default=False Determines whether the gradient with respect to the kernel hyperparameter is determined. Returns ------- - K : array, shape (n_samples_X, n_samples_Y, n_kernels) + K : ndarray of shape (n_samples_X, n_samples_Y, n_kernels) Kernel k(X, Y) - K_gradient : array, shape (n_samples_X, n_samples_X, n_dims, n_kernels) + K_gradient : ndarray of shape \ + (n_samples_X, n_samples_X, n_dims, n_kernels), optional The gradient of the kernel k(X, X) with respect to the - hyperparameter of the kernel. Only returned when eval_gradient + hyperparameter of the kernel. Only returned when `eval_gradient` is True. """ if eval_gradient: @@ -524,21 +548,27 @@ def is_stationary(self): """Returns whether the kernel is stationary. """ return np.all([kernel.is_stationary() for kernel in self.kernels]) + @property + def requires_vector_input(self): + """Returns whether the kernel is defined on discrete structures. """ + return np.any([kernel.requires_vector_input + for kernel in self.kernels]) + def diag(self, X): """Returns the diagonal of the kernel k(X, X). - The result of this method is identical to np.diag(self(X)); however, + The result of this method is identical to `np.diag(self(X))`; however, it can be evaluated more efficiently since only the diagonal is evaluated. Parameters ---------- - X : array, shape (n_samples_X, n_features) - Left argument of the returned kernel k(X, Y) + X : array-like of shape (n_samples_X, n_features) or list of object + Argument to the kernel. Returns ------- - K_diag : array, shape (n_samples_X, n_kernels) + K_diag : ndarray of shape (n_samples_X, n_kernels) Diagonal of kernel k(X, X) """ return np.vstack([kernel.diag(X) for kernel in self.kernels]).T @@ -559,13 +589,13 @@ def get_params(self, deep=True): Parameters ---------- - deep : boolean, optional + deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns ------- - params : mapping of string to any + params : dict Parameter names mapped to their values. """ params = dict(k1=self.k1, k2=self.k2) @@ -603,7 +633,7 @@ def theta(self): Returns ------- - theta : array, shape (n_dims,) + theta : ndarray of shape (n_dims,) The non-fixed, log-transformed hyperparameters of the kernel """ return np.append(self.k1.theta, self.k2.theta) @@ -614,7 +644,7 @@ def theta(self, theta): Parameters ---------- - theta : array, shape (n_dims,) + theta : ndarray of shape (n_dims,) The non-fixed, log-transformed hyperparameters of the kernel """ k1_dims = self.k1.n_dims @@ -627,7 +657,7 @@ def bounds(self): Returns ------- - bounds : array, shape (n_dims, 2) + bounds : ndarray of shape (n_dims, 2) The log-transformed bounds on the kernel's hyperparameters theta """ if self.k1.bounds.size == 0: @@ -646,23 +676,50 @@ def is_stationary(self): """Returns whether the kernel is stationary. """ return self.k1.is_stationary() and self.k2.is_stationary() + @property + def requires_vector_input(self): + """Returns whether the kernel is stationary. """ + return (self.k1.requires_vector_input or + self.k2.requires_vector_input) + class Sum(KernelOperator): - """Sum-kernel k1 + k2 of two kernels k1 and k2. + """The `Sum` kernel takes two kernels :math:`k_1` and :math:`k_2` + and combines them via + + .. math:: + k_{sum}(X, Y) = k_1(X, Y) + k_2(X, Y) + + Note that the `__add__` magic method is overridden, so + `Sum(RBF(), RBF())` is equivalent to using the + operator + with `RBF() + RBF()`. + - The resulting kernel is defined as - k_sum(X, Y) = k1(X, Y) + k2(X, Y) + Read more in the :ref:`User Guide `. .. versionadded:: 0.18 Parameters ---------- - k1 : Kernel object + k1 : Kernel The first base-kernel of the sum-kernel - k2 : Kernel object + k2 : Kernel The second base-kernel of the sum-kernel + Examples + -------- + >>> from sklearn.datasets import make_friedman2 + >>> from sklearn.gaussian_process import GaussianProcessRegressor + >>> from sklearn.gaussian_process.kernels import RBF, Sum, ConstantKernel + >>> X, y = make_friedman2(n_samples=500, noise=0, random_state=0) + >>> kernel = Sum(ConstantKernel(2), RBF()) + >>> gpr = GaussianProcessRegressor(kernel=kernel, + ... random_state=0).fit(X, y) + >>> gpr.score(X, y) + 1.0 + >>> kernel + 1.41**2 + RBF(length_scale=1) """ def __call__(self, X, Y=None, eval_gradient=False): @@ -670,25 +727,27 @@ def __call__(self, X, Y=None, eval_gradient=False): Parameters ---------- - X : array, shape (n_samples_X, n_features) + X : array-like of shape (n_samples_X, n_features) or list of object Left argument of the returned kernel k(X, Y) - Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Y : array-like of shape (n_samples_X, n_features) or list of object,\ + default=None Right argument of the returned kernel k(X, Y). If None, k(X, X) - if evaluated instead. + is evaluated instead. - eval_gradient : bool (optional, default=False) + eval_gradient : bool, default=False Determines whether the gradient with respect to the kernel hyperparameter is determined. Returns ------- - K : array, shape (n_samples_X, n_samples_Y) + K : ndarray of shape (n_samples_X, n_samples_Y) Kernel k(X, Y) - K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + K_gradient : ndarray of shape (n_samples_X, n_samples_X, n_dims),\ + optional The gradient of the kernel k(X, X) with respect to the - hyperparameter of the kernel. Only returned when eval_gradient + hyperparameter of the kernel. Only returned when `eval_gradient` is True. """ if eval_gradient: @@ -701,18 +760,18 @@ def __call__(self, X, Y=None, eval_gradient=False): def diag(self, X): """Returns the diagonal of the kernel k(X, X). - The result of this method is identical to np.diag(self(X)); however, + The result of this method is identical to `np.diag(self(X))`; however, it can be evaluated more efficiently since only the diagonal is evaluated. Parameters ---------- - X : array, shape (n_samples_X, n_features) - Left argument of the returned kernel k(X, Y) + X : array-like of shape (n_samples_X, n_features) or list of object + Argument to the kernel. Returns ------- - K_diag : array, shape (n_samples_X,) + K_diag : ndarray of shape (n_samples_X,) Diagonal of kernel k(X, X) """ return self.k1.diag(X) + self.k2.diag(X) @@ -722,21 +781,43 @@ def __repr__(self): class Product(KernelOperator): - """Product-kernel k1 * k2 of two kernels k1 and k2. + """The `Product` kernel takes two kernels :math:`k_1` and :math:`k_2` + and combines them via + + .. math:: + k_{prod}(X, Y) = k_1(X, Y) * k_2(X, Y) + + Note that the `__mul__` magic method is overridden, so + `Product(RBF(), RBF())` is equivalent to using the * operator + with `RBF() * RBF()`. - The resulting kernel is defined as - k_prod(X, Y) = k1(X, Y) * k2(X, Y) + Read more in the :ref:`User Guide `. .. versionadded:: 0.18 Parameters ---------- - k1 : Kernel object + k1 : Kernel The first base-kernel of the product-kernel - k2 : Kernel object + k2 : Kernel The second base-kernel of the product-kernel + + Examples + -------- + >>> from sklearn.datasets import make_friedman2 + >>> from sklearn.gaussian_process import GaussianProcessRegressor + >>> from sklearn.gaussian_process.kernels import (RBF, Product, + ... ConstantKernel) + >>> X, y = make_friedman2(n_samples=500, noise=0, random_state=0) + >>> kernel = Product(ConstantKernel(2), RBF()) + >>> gpr = GaussianProcessRegressor(kernel=kernel, + ... random_state=0).fit(X, y) + >>> gpr.score(X, y) + 1.0 + >>> kernel + 1.41**2 * RBF(length_scale=1) """ def __call__(self, X, Y=None, eval_gradient=False): @@ -744,25 +825,27 @@ def __call__(self, X, Y=None, eval_gradient=False): Parameters ---------- - X : array, shape (n_samples_X, n_features) + X : array-like of shape (n_samples_X, n_features) or list of object Left argument of the returned kernel k(X, Y) - Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Y : array-like of shape (n_samples_Y, n_features) or list of object,\ + default=None Right argument of the returned kernel k(X, Y). If None, k(X, X) - if evaluated instead. + is evaluated instead. - eval_gradient : bool (optional, default=False) + eval_gradient : bool, default=False Determines whether the gradient with respect to the kernel hyperparameter is determined. Returns ------- - K : array, shape (n_samples_X, n_samples_Y) + K : ndarray of shape (n_samples_X, n_samples_Y) Kernel k(X, Y) - K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + K_gradient : ndarray of shape (n_samples_X, n_samples_X, n_dims), \ + optional The gradient of the kernel k(X, X) with respect to the - hyperparameter of the kernel. Only returned when eval_gradient + hyperparameter of the kernel. Only returned when `eval_gradient` is True. """ if eval_gradient: @@ -782,12 +865,12 @@ def diag(self, X): Parameters ---------- - X : array, shape (n_samples_X, n_features) - Left argument of the returned kernel k(X, Y) + X : array-like of shape (n_samples_X, n_features) or list of object + Argument to the kernel. Returns ------- - K_diag : array, shape (n_samples_X,) + K_diag : ndarray of shape (n_samples_X,) Diagonal of kernel k(X, X) """ return self.k1.diag(X) * self.k2.diag(X) @@ -797,21 +880,44 @@ def __repr__(self): class Exponentiation(Kernel): - """Exponentiate kernel by given exponent. + """The Exponentiation kernel takes one base kernel and a scalar parameter + :math:`p` and combines them via + + .. math:: + k_{exp}(X, Y) = k(X, Y) ^p - The resulting kernel is defined as - k_exp(X, Y) = k(X, Y) ** exponent + Note that the `__pow__` magic method is overridden, so + `Exponentiation(RBF(), 2)` is equivalent to using the ** operator + with `RBF() ** 2`. + + + Read more in the :ref:`User Guide `. .. versionadded:: 0.18 Parameters ---------- - kernel : Kernel object + kernel : Kernel The base kernel exponent : float The exponent for the base kernel + + Examples + -------- + >>> from sklearn.datasets import make_friedman2 + >>> from sklearn.gaussian_process import GaussianProcessRegressor + >>> from sklearn.gaussian_process.kernels import (RationalQuadratic, + ... Exponentiation) + >>> X, y = make_friedman2(n_samples=500, noise=0, random_state=0) + >>> kernel = Exponentiation(RationalQuadratic(), exponent=2) + >>> gpr = GaussianProcessRegressor(kernel=kernel, alpha=5, + ... random_state=0).fit(X, y) + >>> gpr.score(X, y) + 0.419... + >>> gpr.predict(X[:1,:], return_std=True) + (array([635.5...]), array([0.559...])) """ def __init__(self, kernel, exponent): self.kernel = kernel @@ -822,13 +928,13 @@ def get_params(self, deep=True): Parameters ---------- - deep : boolean, optional + deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns ------- - params : mapping of string to any + params : dict Parameter names mapped to their values. """ params = dict(kernel=self.kernel, exponent=self.exponent) @@ -859,7 +965,7 @@ def theta(self): Returns ------- - theta : array, shape (n_dims,) + theta : ndarray of shape (n_dims,) The non-fixed, log-transformed hyperparameters of the kernel """ return self.kernel.theta @@ -870,7 +976,7 @@ def theta(self, theta): Parameters ---------- - theta : array, shape (n_dims,) + theta : ndarray of shape (n_dims,) The non-fixed, log-transformed hyperparameters of the kernel """ self.kernel.theta = theta @@ -881,7 +987,7 @@ def bounds(self): Returns ------- - bounds : array, shape (n_dims, 2) + bounds : ndarray of shape (n_dims, 2) The log-transformed bounds on the kernel's hyperparameters theta """ return self.kernel.bounds @@ -896,25 +1002,27 @@ def __call__(self, X, Y=None, eval_gradient=False): Parameters ---------- - X : array, shape (n_samples_X, n_features) + X : array-like of shape (n_samples_X, n_features) or list of object Left argument of the returned kernel k(X, Y) - Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Y : array-like of shape (n_samples_Y, n_features) or list of object,\ + default=None Right argument of the returned kernel k(X, Y). If None, k(X, X) - if evaluated instead. + is evaluated instead. - eval_gradient : bool (optional, default=False) + eval_gradient : bool, default=False Determines whether the gradient with respect to the kernel hyperparameter is determined. Returns ------- - K : array, shape (n_samples_X, n_samples_Y) + K : ndarray of shape (n_samples_X, n_samples_Y) Kernel k(X, Y) - K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + K_gradient : ndarray of shape (n_samples_X, n_samples_X, n_dims),\ + optional The gradient of the kernel k(X, X) with respect to the - hyperparameter of the kernel. Only returned when eval_gradient + hyperparameter of the kernel. Only returned when `eval_gradient` is True. """ if eval_gradient: @@ -935,12 +1043,12 @@ def diag(self, X): Parameters ---------- - X : array, shape (n_samples_X, n_features) - Left argument of the returned kernel k(X, Y) + X : array-like of shape (n_samples_X, n_features) or list of object + Argument to the kernel. Returns ------- - K_diag : array, shape (n_samples_X,) + K_diag : ndarray of shape (n_samples_X,) Diagonal of kernel k(X, X) """ return self.kernel.diag(X) ** self.exponent @@ -952,27 +1060,60 @@ def is_stationary(self): """Returns whether the kernel is stationary. """ return self.kernel.is_stationary() + @property + def requires_vector_input(self): + """Returns whether the kernel is defined on discrete structures. """ + return self.kernel.requires_vector_input + -class ConstantKernel(StationaryKernelMixin, Kernel): +class ConstantKernel(StationaryKernelMixin, GenericKernelMixin, + Kernel): """Constant kernel. Can be used as part of a product-kernel where it scales the magnitude of the other factor (kernel) or as part of a sum-kernel, where it modifies the mean of the Gaussian process. - k(x_1, x_2) = constant_value for all x_1, x_2 + .. math:: + k(x_1, x_2) = constant\\_value \\;\\forall\\; x_1, x_2 + + Adding a constant kernel is equivalent to adding a constant:: + + kernel = RBF() + ConstantKernel(constant_value=2) + + is the same as:: + + kernel = RBF() + 2 + + + Read more in the :ref:`User Guide `. .. versionadded:: 0.18 Parameters ---------- - constant_value : float, default: 1.0 + constant_value : float, default=1.0 The constant value which defines the covariance: k(x_1, x_2) = constant_value - constant_value_bounds : pair of floats >= 0, default: (1e-5, 1e5) - The lower and upper bound on constant_value - + constant_value_bounds : pair of floats >= 0 or "fixed", default=(1e-5, 1e5) + The lower and upper bound on `constant_value`. + If set to "fixed", `constant_value` cannot be changed during + hyperparameter tuning. + + Examples + -------- + >>> from sklearn.datasets import make_friedman2 + >>> from sklearn.gaussian_process import GaussianProcessRegressor + >>> from sklearn.gaussian_process.kernels import RBF, ConstantKernel + >>> X, y = make_friedman2(n_samples=500, noise=0, random_state=0) + >>> kernel = RBF() + ConstantKernel(constant_value=2) + >>> gpr = GaussianProcessRegressor(kernel=kernel, alpha=5, + ... random_state=0).fit(X, y) + >>> gpr.score(X, y) + 0.3696... + >>> gpr.predict(X[:1,:], return_std=True) + (array([606.1...]), array([0.24...])) """ def __init__(self, constant_value=1.0, constant_value_bounds=(1e-5, 1e5)): self.constant_value = constant_value @@ -988,42 +1129,43 @@ def __call__(self, X, Y=None, eval_gradient=False): Parameters ---------- - X : array, shape (n_samples_X, n_features) + X : array-like of shape (n_samples_X, n_features) or list of object Left argument of the returned kernel k(X, Y) - Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Y : array-like of shape (n_samples_X, n_features) or list of object, \ + default=None Right argument of the returned kernel k(X, Y). If None, k(X, X) - if evaluated instead. + is evaluated instead. - eval_gradient : bool (optional, default=False) + eval_gradient : bool, default=False Determines whether the gradient with respect to the kernel hyperparameter is determined. Only supported when Y is None. Returns ------- - K : array, shape (n_samples_X, n_samples_Y) + K : ndarray of shape (n_samples_X, n_samples_Y) Kernel k(X, Y) - K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + K_gradient : ndarray of shape (n_samples_X, n_samples_X, n_dims), \ + optional The gradient of the kernel k(X, X) with respect to the hyperparameter of the kernel. Only returned when eval_gradient is True. """ - X = np.atleast_2d(X) if Y is None: Y = X elif eval_gradient: raise ValueError("Gradient can only be evaluated when Y is None.") - K = np.full((X.shape[0], Y.shape[0]), self.constant_value, + K = np.full((_num_samples(X), _num_samples(Y)), self.constant_value, dtype=np.array(self.constant_value).dtype) if eval_gradient: if not self.hyperparameter_constant_value.fixed: - return (K, np.full((X.shape[0], X.shape[0], 1), + return (K, np.full((_num_samples(X), _num_samples(X), 1), self.constant_value, dtype=np.array(self.constant_value).dtype)) else: - return K, np.empty((X.shape[0], X.shape[0], 0)) + return K, np.empty((_num_samples(X), _num_samples(X), 0)) else: return K @@ -1036,22 +1178,23 @@ def diag(self, X): Parameters ---------- - X : array, shape (n_samples_X, n_features) - Left argument of the returned kernel k(X, Y) + X : array-like of shape (n_samples_X, n_features) or list of object + Argument to the kernel. Returns ------- - K_diag : array, shape (n_samples_X,) + K_diag : ndarray of shape (n_samples_X,) Diagonal of kernel k(X, X) """ - return np.full(X.shape[0], self.constant_value, + return np.full(_num_samples(X), self.constant_value, dtype=np.array(self.constant_value).dtype) def __repr__(self): return "{0:.3g}**2".format(np.sqrt(self.constant_value)) -class WhiteKernel(StationaryKernelMixin, Kernel): +class WhiteKernel(StationaryKernelMixin, GenericKernelMixin, + Kernel): """White kernel. The main use-case of this kernel is as part of a sum-kernel where it @@ -1059,17 +1202,37 @@ class WhiteKernel(StationaryKernelMixin, Kernel): normally-distributed. The parameter noise_level equals the variance of this noise. - k(x_1, x_2) = noise_level if x_1 == x_2 else 0 + .. math:: + k(x_1, x_2) = noise\\_level \\text{ if } x_i == x_j \\text{ else } 0 + + + Read more in the :ref:`User Guide `. .. versionadded:: 0.18 Parameters ---------- - noise_level : float, default: 1.0 + noise_level : float, default=1.0 Parameter controlling the noise level (variance) - noise_level_bounds : pair of floats >= 0, default: (1e-5, 1e5) - The lower and upper bound on noise_level + noise_level_bounds : pair of floats >= 0 or "fixed", default=(1e-5, 1e5) + The lower and upper bound on 'noise_level'. + If set to "fixed", 'noise_level' cannot be changed during + hyperparameter tuning. + + Examples + -------- + >>> from sklearn.datasets import make_friedman2 + >>> from sklearn.gaussian_process import GaussianProcessRegressor + >>> from sklearn.gaussian_process.kernels import DotProduct, WhiteKernel + >>> X, y = make_friedman2(n_samples=500, noise=0, random_state=0) + >>> kernel = DotProduct() + WhiteKernel(noise_level=0.5) + >>> gpr = GaussianProcessRegressor(kernel=kernel, + ... random_state=0).fit(X, y) + >>> gpr.score(X, y) + 0.3680... + >>> gpr.predict(X[:2,:], return_std=True) + (array([653.0..., 592.1... ]), array([316.6..., 316.6...])) """ def __init__(self, noise_level=1.0, noise_level_bounds=(1e-5, 1e5)): self.noise_level = noise_level @@ -1085,43 +1248,44 @@ def __call__(self, X, Y=None, eval_gradient=False): Parameters ---------- - X : array, shape (n_samples_X, n_features) + X : array-like of shape (n_samples_X, n_features) or list of object Left argument of the returned kernel k(X, Y) - Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Y : array-like of shape (n_samples_X, n_features) or list of object,\ + default=None Right argument of the returned kernel k(X, Y). If None, k(X, X) - if evaluated instead. + is evaluated instead. - eval_gradient : bool (optional, default=False) + eval_gradient : bool, default=False Determines whether the gradient with respect to the kernel hyperparameter is determined. Only supported when Y is None. Returns ------- - K : array, shape (n_samples_X, n_samples_Y) + K : ndarray of shape (n_samples_X, n_samples_Y) Kernel k(X, Y) - K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + K_gradient : ndarray of shape (n_samples_X, n_samples_X, n_dims),\ + optional The gradient of the kernel k(X, X) with respect to the hyperparameter of the kernel. Only returned when eval_gradient is True. """ - X = np.atleast_2d(X) if Y is not None and eval_gradient: raise ValueError("Gradient can only be evaluated when Y is None.") if Y is None: - K = self.noise_level * np.eye(X.shape[0]) + K = self.noise_level * np.eye(_num_samples(X)) if eval_gradient: if not self.hyperparameter_noise_level.fixed: return (K, self.noise_level - * np.eye(X.shape[0])[:, :, np.newaxis]) + * np.eye(_num_samples(X))[:, :, np.newaxis]) else: - return K, np.empty((X.shape[0], X.shape[0], 0)) + return K, np.empty((_num_samples(X), _num_samples(X), 0)) else: return K else: - return np.zeros((X.shape[0], Y.shape[0])) + return np.zeros((_num_samples(X), _num_samples(Y))) def diag(self, X): """Returns the diagonal of the kernel k(X, X). @@ -1132,15 +1296,15 @@ def diag(self, X): Parameters ---------- - X : array, shape (n_samples_X, n_features) - Left argument of the returned kernel k(X, Y) + X : array-like of shape (n_samples_X, n_features) or list of object + Argument to the kernel. Returns ------- - K_diag : array, shape (n_samples_X,) + K_diag : ndarray of shape (n_samples_X,) Diagonal of kernel k(X, X) """ - return np.full(X.shape[0], self.noise_level, + return np.full(_num_samples(X), self.noise_level, dtype=np.array(self.noise_level).dtype) def __repr__(self): @@ -1152,29 +1316,63 @@ class RBF(StationaryKernelMixin, NormalizedKernelMixin, Kernel): """Radial-basis function kernel (aka squared-exponential kernel). The RBF kernel is a stationary kernel. It is also known as the - "squared exponential" kernel. It is parameterized by a length-scale - parameter length_scale>0, which can either be a scalar (isotropic variant + "squared exponential" kernel. It is parameterized by a length scale + parameter :math:`l>0`, which can either be a scalar (isotropic variant of the kernel) or a vector with the same number of dimensions as the inputs X (anisotropic variant of the kernel). The kernel is given by: - k(x_i, x_j) = exp(-1 / 2 d(x_i / length_scale, x_j / length_scale)^2) + .. math:: + k(x_i, x_j) = \\exp\\left(- \\frac{d(x_i, x_j)^2}{2l^2} \\right) + + where :math:`l` is the length scale of the kernel and + :math:`d(\\cdot,\\cdot)` is the Euclidean distance. + For advice on how to set the length scale parameter, see e.g. [1]_. This kernel is infinitely differentiable, which implies that GPs with this kernel as covariance function have mean square derivatives of all orders, and are thus very smooth. + See [2]_, Chapter 4, Section 4.2, for further details of the RBF kernel. + + Read more in the :ref:`User Guide `. .. versionadded:: 0.18 Parameters ---------- - length_scale : float or array with shape (n_features,), default: 1.0 + length_scale : float or ndarray of shape (n_features,), default=1.0 The length scale of the kernel. If a float, an isotropic kernel is used. If an array, an anisotropic kernel is used where each dimension of l defines the length-scale of the respective feature dimension. - length_scale_bounds : pair of floats >= 0, default: (1e-5, 1e5) - The lower and upper bound on length_scale + length_scale_bounds : pair of floats >= 0 or "fixed", default=(1e-5, 1e5) + The lower and upper bound on 'length_scale'. + If set to "fixed", 'length_scale' cannot be changed during + hyperparameter tuning. + References + ---------- + .. [1] `David Duvenaud (2014). "The Kernel Cookbook: + Advice on Covariance functions". + `_ + + .. [2] `Carl Edward Rasmussen, Christopher K. I. Williams (2006). + "Gaussian Processes for Machine Learning". The MIT Press. + `_ + + Examples + -------- + >>> from sklearn.datasets import load_iris + >>> from sklearn.gaussian_process import GaussianProcessClassifier + >>> from sklearn.gaussian_process.kernels import RBF + >>> X, y = load_iris(return_X_y=True) + >>> kernel = 1.0 * RBF(1.0) + >>> gpc = GaussianProcessClassifier(kernel=kernel, + ... random_state=0).fit(X, y) + >>> gpc.score(X, y) + 0.9866... + >>> gpc.predict_proba(X[:2,:]) + array([[0.8354..., 0.03228..., 0.1322...], + [0.7906..., 0.0652..., 0.1441...]]) """ def __init__(self, length_scale=1.0, length_scale_bounds=(1e-5, 1e5)): self.length_scale = length_scale @@ -1198,25 +1396,26 @@ def __call__(self, X, Y=None, eval_gradient=False): Parameters ---------- - X : array, shape (n_samples_X, n_features) + X : ndarray of shape (n_samples_X, n_features) Left argument of the returned kernel k(X, Y) - Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Y : ndarray of shape (n_samples_Y, n_features), default=None Right argument of the returned kernel k(X, Y). If None, k(X, X) if evaluated instead. - eval_gradient : bool (optional, default=False) + eval_gradient : bool, default=False Determines whether the gradient with respect to the kernel hyperparameter is determined. Only supported when Y is None. Returns ------- - K : array, shape (n_samples_X, n_samples_Y) + K : ndarray of shape (n_samples_X, n_samples_Y) Kernel k(X, Y) - K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + K_gradient : ndarray of shape (n_samples_X, n_samples_X, n_dims), \ + optional The gradient of the kernel k(X, X) with respect to the - hyperparameter of the kernel. Only returned when eval_gradient + hyperparameter of the kernel. Only returned when `eval_gradient` is True. """ X = np.atleast_2d(X) @@ -1265,30 +1464,50 @@ def __repr__(self): class Matern(RBF): """ Matern kernel. - The class of Matern kernels is a generalization of the RBF and the - absolute exponential kernel parameterized by an additional parameter - nu. The smaller nu, the less smooth the approximated function is. - For nu=inf, the kernel becomes equivalent to the RBF kernel and for nu=0.5 - to the absolute exponential kernel. Important intermediate values are - nu=1.5 (once differentiable functions) and nu=2.5 (twice differentiable - functions). + The class of Matern kernels is a generalization of the :class:`RBF`. + It has an additional parameter :math:`\\nu` which controls the + smoothness of the resulting function. The smaller :math:`\\nu`, + the less smooth the approximated function is. + As :math:`\\nu\\rightarrow\\infty`, the kernel becomes equivalent to + the :class:`RBF` kernel. When :math:`\\nu = 1/2`, the Matérn kernel + becomes identical to the absolute exponential kernel. + Important intermediate values are + :math:`\\nu=1.5` (once differentiable functions) + and :math:`\\nu=2.5` (twice differentiable functions). + + The kernel is given by: + + .. math:: + k(x_i, x_j) = \\frac{1}{\\Gamma(\\nu)2^{\\nu-1}}\\Bigg( + \\frac{\\sqrt{2\\nu}}{l} d(x_i , x_j ) + \\Bigg)^\\nu K_\\nu\\Bigg( + \\frac{\\sqrt{2\\nu}}{l} d(x_i , x_j )\\Bigg) + + - See Rasmussen and Williams 2006, pp84 for details regarding the - different variants of the Matern kernel. + where :math:`d(\\cdot,\\cdot)` is the Euclidean distance, + :math:`K_{\\nu}(\\cdot)` is a modified Bessel function and + :math:`\\Gamma(\\cdot)` is the gamma function. + See [1]_, Chapter 4, Section 4.2, for details regarding the different + variants of the Matern kernel. + + Read more in the :ref:`User Guide `. .. versionadded:: 0.18 Parameters ---------- - length_scale : float or array with shape (n_features,), default: 1.0 + length_scale : float or ndarray of shape (n_features,), default=1.0 The length scale of the kernel. If a float, an isotropic kernel is used. If an array, an anisotropic kernel is used where each dimension of l defines the length-scale of the respective feature dimension. - length_scale_bounds : pair of floats >= 0, default: (1e-5, 1e5) - The lower and upper bound on length_scale + length_scale_bounds : pair of floats >= 0 or "fixed", default=(1e-5, 1e5) + The lower and upper bound on 'length_scale'. + If set to "fixed", 'length_scale' cannot be changed during + hyperparameter tuning. - nu : float, default: 1.5 + nu : float, default=1.5 The parameter nu controlling the smoothness of the learned function. The smaller nu, the less smooth the approximated function is. For nu=inf, the kernel becomes equivalent to the RBF kernel and for @@ -1300,6 +1519,26 @@ class Matern(RBF): Bessel function. Furthermore, in contrast to l, nu is kept fixed to its initial value and not optimized. + References + ---------- + .. [1] `Carl Edward Rasmussen, Christopher K. I. Williams (2006). + "Gaussian Processes for Machine Learning". The MIT Press. + `_ + + Examples + -------- + >>> from sklearn.datasets import load_iris + >>> from sklearn.gaussian_process import GaussianProcessClassifier + >>> from sklearn.gaussian_process.kernels import Matern + >>> X, y = load_iris(return_X_y=True) + >>> kernel = 1.0 * Matern(length_scale=1.0, nu=1.5) + >>> gpc = GaussianProcessClassifier(kernel=kernel, + ... random_state=0).fit(X, y) + >>> gpc.score(X, y) + 0.9866... + >>> gpc.predict_proba(X[:2,:]) + array([[0.8513..., 0.0368..., 0.1117...], + [0.8086..., 0.0693..., 0.1220...]]) """ def __init__(self, length_scale=1.0, length_scale_bounds=(1e-5, 1e5), nu=1.5): @@ -1311,25 +1550,26 @@ def __call__(self, X, Y=None, eval_gradient=False): Parameters ---------- - X : array, shape (n_samples_X, n_features) + X : ndarray of shape (n_samples_X, n_features) Left argument of the returned kernel k(X, Y) - Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Y : ndarray of shape (n_samples_Y, n_features), default=None Right argument of the returned kernel k(X, Y). If None, k(X, X) if evaluated instead. - eval_gradient : bool (optional, default=False) + eval_gradient : bool, default=False Determines whether the gradient with respect to the kernel hyperparameter is determined. Only supported when Y is None. Returns ------- - K : array, shape (n_samples_X, n_samples_Y) + K : ndarray of shape (n_samples_X, n_samples_Y) Kernel k(X, Y) - K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + K_gradient : ndarray of shape (n_samples_X, n_samples_X, n_dims), \ + optional The gradient of the kernel k(X, X) with respect to the - hyperparameter of the kernel. Only returned when eval_gradient + hyperparameter of the kernel. Only returned when `eval_gradient` is True. """ X = np.atleast_2d(X) @@ -1351,6 +1591,8 @@ def __call__(self, X, Y=None, eval_gradient=False): elif self.nu == 2.5: K = dists * math.sqrt(5) K = (1. + K + K ** 2 / 3.0) * np.exp(-K) + elif self.nu == np.inf: + K = np.exp(-dists ** 2 / 2.0) else: # general case; expensive to evaluate K = dists K[K == 0.0] += np.finfo(float).eps # strict zeros result in nan @@ -1387,6 +1629,8 @@ def __call__(self, X, Y=None, eval_gradient=False): elif self.nu == 2.5: tmp = np.sqrt(5 * D.sum(-1))[..., np.newaxis] K_gradient = 5.0 / 3.0 * D * (tmp + 1) * np.exp(-tmp) + elif self.nu == np.inf: + K_gradient = D * K[..., np.newaxis] else: # approximate gradient numerically def f(theta): # helper function @@ -1416,29 +1660,63 @@ class RationalQuadratic(StationaryKernelMixin, NormalizedKernelMixin, Kernel): """Rational Quadratic kernel. The RationalQuadratic kernel can be seen as a scale mixture (an infinite - sum) of RBF kernels with different characteristic length-scales. It is - parameterized by a length-scale parameter length_scale>0 and a scale - mixture parameter alpha>0. Only the isotropic variant where length_scale is - a scalar is supported at the moment. The kernel given by: + sum) of RBF kernels with different characteristic length scales. It is + parameterized by a length scale parameter :math:`l>0` and a scale + mixture parameter :math:`\\alpha>0`. Only the isotropic variant + where length_scale :math:`l` is a scalar is supported at the moment. + The kernel is given by: - k(x_i, x_j) = (1 + d(x_i, x_j)^2 / (2*alpha * length_scale^2))^-alpha + .. math:: + k(x_i, x_j) = \\left( + 1 + \\frac{d(x_i, x_j)^2 }{ 2\\alpha l^2}\\right)^{-\\alpha} + + where :math:`\\alpha` is the scale mixture parameter, :math:`l` is + the length scale of the kernel and :math:`d(\\cdot,\\cdot)` is the + Euclidean distance. + For advice on how to set the parameters, see e.g. [1]_. + + Read more in the :ref:`User Guide `. .. versionadded:: 0.18 Parameters ---------- - length_scale : float > 0, default: 1.0 + length_scale : float > 0, default=1.0 The length scale of the kernel. - alpha : float > 0, default: 1.0 + alpha : float > 0, default=1.0 Scale mixture parameter - length_scale_bounds : pair of floats >= 0, default: (1e-5, 1e5) - The lower and upper bound on length_scale + length_scale_bounds : pair of floats >= 0 or "fixed", default=(1e-5, 1e5) + The lower and upper bound on 'length_scale'. + If set to "fixed", 'length_scale' cannot be changed during + hyperparameter tuning. - alpha_bounds : pair of floats >= 0, default: (1e-5, 1e5) - The lower and upper bound on alpha + alpha_bounds : pair of floats >= 0 or "fixed", default=(1e-5, 1e5) + The lower and upper bound on 'alpha'. + If set to "fixed", 'alpha' cannot be changed during + hyperparameter tuning. + References + ---------- + .. [1] `David Duvenaud (2014). "The Kernel Cookbook: + Advice on Covariance functions". + `_ + + Examples + -------- + >>> from sklearn.datasets import load_iris + >>> from sklearn.gaussian_process import GaussianProcessClassifier + >>> from sklearn.gaussian_process.kernels import Matern + >>> X, y = load_iris(return_X_y=True) + >>> kernel = RationalQuadratic(length_scale=1.0, alpha=1.5) + >>> gpc = GaussianProcessClassifier(kernel=kernel, + ... random_state=0).fit(X, y) + >>> gpc.score(X, y) + 0.9733... + >>> gpc.predict_proba(X[:2,:]) + array([[0.8881..., 0.0566..., 0.05518...], + [0.8678..., 0.0707... , 0.0614...]]) """ def __init__(self, length_scale=1.0, alpha=1.0, length_scale_bounds=(1e-5, 1e5), alpha_bounds=(1e-5, 1e5)): @@ -1461,23 +1739,23 @@ def __call__(self, X, Y=None, eval_gradient=False): Parameters ---------- - X : array, shape (n_samples_X, n_features) + X : ndarray of shape (n_samples_X, n_features) Left argument of the returned kernel k(X, Y) - Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Y : ndarray of shape (n_samples_Y, n_features), default=None Right argument of the returned kernel k(X, Y). If None, k(X, X) if evaluated instead. - eval_gradient : bool (optional, default=False) + eval_gradient : bool, default=False Determines whether the gradient with respect to the kernel hyperparameter is determined. Only supported when Y is None. Returns ------- - K : array, shape (n_samples_X, n_samples_Y) + K : ndarray of shape (n_samples_X, n_samples_Y) Kernel k(X, Y) - K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + K_gradient : ndarray of shape (n_samples_X, n_samples_X, n_dims) The gradient of the kernel k(X, X) with respect to the hyperparameter of the kernel. Only returned when eval_gradient is True. @@ -1529,32 +1807,58 @@ def __repr__(self): class ExpSineSquared(StationaryKernelMixin, NormalizedKernelMixin, Kernel): - r"""Exp-Sine-Squared kernel. + r"""Exp-Sine-Squared kernel (aka periodic kernel). - The ExpSineSquared kernel allows modeling periodic functions. It is - parameterized by a length-scale parameter length_scale>0 and a periodicity - parameter periodicity>0. Only the isotropic variant where l is a scalar is - supported at the moment. The kernel given by: + The ExpSineSquared kernel allows one to model functions which repeat + themselves exactly. It is parameterized by a length scale + parameter :math:`l>0` and a periodicity parameter :math:`p>0`. + Only the isotropic variant where :math:`l` is a scalar is + supported at the moment. The kernel is given by: - k(x_i, x_j) = - exp(-2 (sin(\pi / periodicity * d(x_i, x_j)) / length_scale) ^ 2) + .. math:: + k(x_i, x_j) = \text{exp}\left(- + \frac{ 2\sin^2(\pi d(x_i, x_j)/p) }{ l^ 2} \right) + + where :math:`l` is the length scale of the kernel, :math:`p` the + periodicity of the kernel and :math:`d(\\cdot,\\cdot)` is the + Euclidean distance. + + Read more in the :ref:`User Guide `. .. versionadded:: 0.18 Parameters ---------- - length_scale : float > 0, default: 1.0 + + length_scale : float > 0, default=1.0 The length scale of the kernel. - periodicity : float > 0, default: 1.0 + periodicity : float > 0, default=1.0 The periodicity of the kernel. - length_scale_bounds : pair of floats >= 0, default: (1e-5, 1e5) - The lower and upper bound on length_scale - - periodicity_bounds : pair of floats >= 0, default: (1e-5, 1e5) - The lower and upper bound on periodicity - + length_scale_bounds : pair of floats >= 0 or "fixed", default=(1e-5, 1e5) + The lower and upper bound on 'length_scale'. + If set to "fixed", 'length_scale' cannot be changed during + hyperparameter tuning. + + periodicity_bounds : pair of floats >= 0 or "fixed", default=(1e-5, 1e5) + The lower and upper bound on 'periodicity'. + If set to "fixed", 'periodicity' cannot be changed during + hyperparameter tuning. + + Examples + -------- + >>> from sklearn.datasets import make_friedman2 + >>> from sklearn.gaussian_process import GaussianProcessRegressor + >>> from sklearn.gaussian_process.kernels import ExpSineSquared + >>> X, y = make_friedman2(n_samples=50, noise=0, random_state=0) + >>> kernel = ExpSineSquared(length_scale=1, periodicity=1) + >>> gpr = GaussianProcessRegressor(kernel=kernel, alpha=5, + ... random_state=0).fit(X, y) + >>> gpr.score(X, y) + 0.0144... + >>> gpr.predict(X[:2,:], return_std=True) + (array([425.6..., 457.5...]), array([0.3894..., 0.3467...])) """ def __init__(self, length_scale=1.0, periodicity=1.0, length_scale_bounds=(1e-5, 1e5), @@ -1566,6 +1870,7 @@ def __init__(self, length_scale=1.0, periodicity=1.0, @property def hyperparameter_length_scale(self): + """Returns the length scale""" return Hyperparameter( "length_scale", "numeric", self.length_scale_bounds) @@ -1579,25 +1884,26 @@ def __call__(self, X, Y=None, eval_gradient=False): Parameters ---------- - X : array, shape (n_samples_X, n_features) + X : ndarray of shape (n_samples_X, n_features) Left argument of the returned kernel k(X, Y) - Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Y : ndarray of shape (n_samples_Y, n_features), default=None Right argument of the returned kernel k(X, Y). If None, k(X, X) if evaluated instead. - eval_gradient : bool (optional, default=False) + eval_gradient : bool, default=False Determines whether the gradient with respect to the kernel hyperparameter is determined. Only supported when Y is None. Returns ------- - K : array, shape (n_samples_X, n_samples_Y) + K : ndarray of shape (n_samples_X, n_samples_Y) Kernel k(X, Y) - K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + K_gradient : ndarray of shape (n_samples_X, n_samples_X, n_dims), \ + optional The gradient of the kernel k(X, X) with respect to the - hyperparameter of the kernel. Only returned when eval_gradient + hyperparameter of the kernel. Only returned when `eval_gradient` is True. """ X = np.atleast_2d(X) @@ -1645,28 +1951,57 @@ class DotProduct(Kernel): r"""Dot-Product kernel. The DotProduct kernel is non-stationary and can be obtained from linear - regression by putting N(0, 1) priors on the coefficients of x_d (d = 1, . . - . , D) and a prior of N(0, \sigma_0^2) on the bias. The DotProduct kernel - is invariant to a rotation of the coordinates about the origin, but not - translations. It is parameterized by a parameter sigma_0^2. For - sigma_0^2 =0, the kernel is called the homogeneous linear kernel, otherwise + regression by putting :math:`N(0, 1)` priors on the coefficients + of :math:`x_d (d = 1, . . . , D)` and a prior of :math:`N(0, \sigma_0^2)` + on the bias. The DotProduct kernel is invariant to a rotation of + the coordinates about the origin, but not translations. + It is parameterized by a parameter sigma_0 :math:`\sigma` + which controls the inhomogenity of the kernel. For :math:`\sigma_0^2 =0`, + the kernel is called the homogeneous linear kernel, otherwise it is inhomogeneous. The kernel is given by - k(x_i, x_j) = sigma_0 ^ 2 + x_i \cdot x_j + .. math:: + k(x_i, x_j) = \sigma_0 ^ 2 + x_i \cdot x_j The DotProduct kernel is commonly combined with exponentiation. + See [1]_, Chapter 4, Section 4.2, for further details regarding the + DotProduct kernel. + + Read more in the :ref:`User Guide `. + .. versionadded:: 0.18 Parameters ---------- - sigma_0 : float >= 0, default: 1.0 + sigma_0 : float >= 0, default=1.0 Parameter controlling the inhomogenity of the kernel. If sigma_0=0, the kernel is homogenous. - sigma_0_bounds : pair of floats >= 0, default: (1e-5, 1e5) - The lower and upper bound on l + sigma_0_bounds : pair of floats >= 0 or "fixed", default=(1e-5, 1e5) + The lower and upper bound on 'sigma_0'. + If set to "fixed", 'sigma_0' cannot be changed during + hyperparameter tuning. + References + ---------- + .. [1] `Carl Edward Rasmussen, Christopher K. I. Williams (2006). + "Gaussian Processes for Machine Learning". The MIT Press. + `_ + + Examples + -------- + >>> from sklearn.datasets import make_friedman2 + >>> from sklearn.gaussian_process import GaussianProcessRegressor + >>> from sklearn.gaussian_process.kernels import DotProduct, WhiteKernel + >>> X, y = make_friedman2(n_samples=500, noise=0, random_state=0) + >>> kernel = DotProduct() + WhiteKernel() + >>> gpr = GaussianProcessRegressor(kernel=kernel, + ... random_state=0).fit(X, y) + >>> gpr.score(X, y) + 0.3680... + >>> gpr.predict(X[:2,:], return_std=True) + (array([653.0..., 592.1...]), array([316.6..., 316.6...])) """ def __init__(self, sigma_0=1.0, sigma_0_bounds=(1e-5, 1e5)): @@ -1682,25 +2017,26 @@ def __call__(self, X, Y=None, eval_gradient=False): Parameters ---------- - X : array, shape (n_samples_X, n_features) + X : ndarray of shape (n_samples_X, n_features) Left argument of the returned kernel k(X, Y) - Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Y : ndarray of shape (n_samples_Y, n_features), default=None Right argument of the returned kernel k(X, Y). If None, k(X, X) if evaluated instead. - eval_gradient : bool (optional, default=False) + eval_gradient : bool, default=False Determines whether the gradient with respect to the kernel hyperparameter is determined. Only supported when Y is None. Returns ------- - K : array, shape (n_samples_X, n_samples_Y) + K : ndarray of shape (n_samples_X, n_samples_Y) Kernel k(X, Y) - K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + K_gradient : ndarray of shape (n_samples_X, n_samples_X, n_dims),\ + optional The gradient of the kernel k(X, X) with respect to the - hyperparameter of the kernel. Only returned when eval_gradient + hyperparameter of the kernel. Only returned when `eval_gradient` is True. """ X = np.atleast_2d(X) @@ -1731,13 +2067,13 @@ def diag(self, X): Parameters ---------- - X : array, shape (n_samples_X, n_features) - Left argument of the returned kernel k(X, Y) + X : ndarray of shape (n_samples_X, n_features) + Left argument of the returned kernel k(X, Y). Returns ------- - K_diag : array, shape (n_samples_X,) - Diagonal of kernel k(X, X) + K_diag : ndarray of shape (n_samples_X,) + Diagonal of kernel k(X, X). """ return np.einsum('ij,ij->i', X, X) + self.sigma_0 ** 2 @@ -1779,13 +2115,18 @@ class PairwiseKernel(Kernel): Parameters ---------- - gamma : float >= 0, default: 1.0 - Parameter gamma of the pairwise kernel specified by metric - - gamma_bounds : pair of floats >= 0, default: (1e-5, 1e5) - The lower and upper bound on gamma - - metric : string, or callable, default: "linear" + gamma : float, default=1.0 + Parameter gamma of the pairwise kernel specified by metric. It should + be positive. + + gamma_bounds : pair of floats >= 0 or "fixed", default=(1e-5, 1e5) + The lower and upper bound on 'gamma'. + If set to "fixed", 'gamma' cannot be changed during + hyperparameter tuning. + + metric : {"linear", "additive_chi2", "chi2", "poly", "polynomial", \ + "rbf", "laplacian", "sigmoid", "cosine"} or callable, \ + default="linear" The metric to use when calculating kernel between instances in a feature array. If metric is a string, it must be one of the metrics in pairwise.PAIRWISE_KERNEL_FUNCTIONS. @@ -1795,7 +2136,7 @@ class PairwiseKernel(Kernel): should take two arrays from X as input and return a value indicating the distance between them. - pairwise_kernels_kwargs : dict, default: None + pairwise_kernels_kwargs : dict, default=None All entries of this dict (if any) are passed as keyword arguments to the pairwise kernel function. @@ -1817,25 +2158,26 @@ def __call__(self, X, Y=None, eval_gradient=False): Parameters ---------- - X : array, shape (n_samples_X, n_features) + X : ndarray of shape (n_samples_X, n_features) Left argument of the returned kernel k(X, Y) - Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Y : ndarray of shape (n_samples_Y, n_features), default=None Right argument of the returned kernel k(X, Y). If None, k(X, X) if evaluated instead. - eval_gradient : bool (optional, default=False) + eval_gradient : bool, default=False Determines whether the gradient with respect to the kernel hyperparameter is determined. Only supported when Y is None. Returns ------- - K : array, shape (n_samples_X, n_samples_Y) + K : ndarray of shape (n_samples_X, n_samples_Y) Kernel k(X, Y) - K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + K_gradient : ndarray of shape (n_samples_X, n_samples_X, n_dims),\ + optional The gradient of the kernel k(X, X) with respect to the - hyperparameter of the kernel. Only returned when eval_gradient + hyperparameter of the kernel. Only returned when `eval_gradient` is True. """ pairwise_kernels_kwargs = self.pairwise_kernels_kwargs @@ -1868,12 +2210,12 @@ def diag(self, X): Parameters ---------- - X : array, shape (n_samples_X, n_features) + X : ndarray of shape (n_samples_X, n_features) Left argument of the returned kernel k(X, Y) Returns ------- - K_diag : array, shape (n_samples_X,) + K_diag : ndarray of shape (n_samples_X,) Diagonal of kernel k(X, X) """ # We have to fall back to slow way of computing diagonal diff --git a/sklearn/gaussian_process/tests/_mini_sequence_kernel.py b/sklearn/gaussian_process/tests/_mini_sequence_kernel.py new file mode 100644 index 0000000000000..c260a361e1e71 --- /dev/null +++ b/sklearn/gaussian_process/tests/_mini_sequence_kernel.py @@ -0,0 +1,51 @@ +from sklearn.gaussian_process.kernels import Kernel, Hyperparameter +from sklearn.gaussian_process.kernels import GenericKernelMixin +from sklearn.gaussian_process.kernels import StationaryKernelMixin +import numpy as np +from sklearn.base import clone + + +class MiniSeqKernel(GenericKernelMixin, + StationaryKernelMixin, + Kernel): + ''' + A minimal (but valid) convolutional kernel for sequences of variable + length. + ''' + def __init__(self, + baseline_similarity=0.5, + baseline_similarity_bounds=(1e-5, 1)): + self.baseline_similarity = baseline_similarity + self.baseline_similarity_bounds = baseline_similarity_bounds + + @property + def hyperparameter_baseline_similarity(self): + return Hyperparameter("baseline_similarity", + "numeric", + self.baseline_similarity_bounds) + + def _f(self, s1, s2): + return sum([1.0 if c1 == c2 else self.baseline_similarity + for c1 in s1 + for c2 in s2]) + + def _g(self, s1, s2): + return sum([0.0 if c1 == c2 else 1.0 for c1 in s1 for c2 in s2]) + + def __call__(self, X, Y=None, eval_gradient=False): + if Y is None: + Y = X + + if eval_gradient: + return (np.array([[self._f(x, y) for y in Y] for x in X]), + np.array([[[self._g(x, y)] for y in Y] for x in X])) + else: + return np.array([[self._f(x, y) for y in Y] for x in X]) + + def diag(self, X): + return np.array([self._f(x, x) for x in X]) + + def clone_with_theta(self, theta): + cloned = clone(self) + cloned.theta = theta + return cloned diff --git a/sklearn/gaussian_process/tests/test_gpc.py b/sklearn/gaussian_process/tests/test_gpc.py index aec5cc147223f..72d550231f4ea 100644 --- a/sklearn/gaussian_process/tests/test_gpc.py +++ b/sklearn/gaussian_process/tests/test_gpc.py @@ -11,12 +11,15 @@ from sklearn.gaussian_process import GaussianProcessClassifier from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C +from sklearn.gaussian_process.tests._mini_sequence_kernel import MiniSeqKernel from sklearn.utils._testing import assert_almost_equal, assert_array_equal def f(x): return np.sin(x) + + X = np.atleast_2d(np.linspace(0, 10, 30)).T X2 = np.atleast_2d([2., 4., 5.5, 6.5, 7.5]).T y = np.array(f(X).ravel() > 0, dtype=int) @@ -44,12 +47,22 @@ def test_predict_consistent(kernel): gpc.predict_proba(X)[:, 1] >= 0.5) +def test_predict_consistent_structured(): + # Check binary predict decision has also predicted probability above 0.5. + X = ['A', 'AB', 'B'] + y = np.array([True, False, True]) + kernel = MiniSeqKernel(baseline_similarity_bounds='fixed') + gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y) + assert_array_equal(gpc.predict(X), + gpc.predict_proba(X)[:, 1] >= 0.5) + + @pytest.mark.parametrize('kernel', non_fixed_kernels) def test_lml_improving(kernel): # Test that hyperparameter-tuning improves log-marginal likelihood. gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y) assert (gpc.log_marginal_likelihood(gpc.kernel_.theta) > - gpc.log_marginal_likelihood(kernel.theta)) + gpc.log_marginal_likelihood(kernel.theta)) @pytest.mark.parametrize('kernel', kernels) @@ -139,7 +152,7 @@ def optimizer(obj_func, initial_theta, bounds): gpc.fit(X, y_mc) # Checks that optimizer improved marginal likelihood assert (gpc.log_marginal_likelihood(gpc.kernel_.theta) > - gpc.log_marginal_likelihood(kernel.theta)) + gpc.log_marginal_likelihood(kernel.theta)) @pytest.mark.parametrize('kernel', kernels) diff --git a/sklearn/gaussian_process/tests/test_gpr.py b/sklearn/gaussian_process/tests/test_gpr.py index 64b177ce17c48..4bdd94e669eb4 100644 --- a/sklearn/gaussian_process/tests/test_gpr.py +++ b/sklearn/gaussian_process/tests/test_gpr.py @@ -1,8 +1,10 @@ """Testing for Gaussian process regression """ # Author: Jan Hendrik Metzen +# Modified by: Pete Green # License: BSD 3 clause +import sys import numpy as np from scipy.optimize import approx_fprime @@ -13,11 +15,13 @@ from sklearn.gaussian_process.kernels \ import RBF, ConstantKernel as C, WhiteKernel from sklearn.gaussian_process.kernels import DotProduct +from sklearn.gaussian_process.tests._mini_sequence_kernel import MiniSeqKernel from sklearn.utils._testing \ import (assert_array_less, assert_almost_equal, assert_raise_message, - assert_array_almost_equal, assert_array_equal) + assert_array_almost_equal, assert_array_equal, + assert_allclose) def f(x): @@ -45,20 +49,40 @@ def f(x): @pytest.mark.parametrize('kernel', kernels) def test_gpr_interpolation(kernel): + if sys.maxsize <= 2 ** 32 and sys.version_info[:2] == (3, 6): + pytest.xfail("This test may fail on 32bit Py3.6") + + # Test the interpolating property for different kernels. + gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) + y_pred, y_cov = gpr.predict(X, return_cov=True) + + assert_almost_equal(y_pred, y) + assert_almost_equal(np.diag(y_cov), 0.) + + +def test_gpr_interpolation_structured(): # Test the interpolating property for different kernels. + kernel = MiniSeqKernel(baseline_similarity_bounds='fixed') + X = ['A', 'B', 'C'] + y = np.array([1, 2, 3]) gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) y_pred, y_cov = gpr.predict(X, return_cov=True) + assert_almost_equal(kernel(X, eval_gradient=True)[1].ravel(), + (1 - np.eye(len(X))).ravel()) assert_almost_equal(y_pred, y) assert_almost_equal(np.diag(y_cov), 0.) @pytest.mark.parametrize('kernel', non_fixed_kernels) def test_lml_improving(kernel): + if sys.maxsize <= 2 ** 32 and sys.version_info[:2] == (3, 6): + pytest.xfail("This test may fail on 32bit Py3.6") + # Test that hyperparameter-tuning improves log-marginal likelihood. gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) assert (gpr.log_marginal_likelihood(gpr.kernel_.theta) > - gpr.log_marginal_likelihood(kernel.theta)) + gpr.log_marginal_likelihood(kernel.theta)) @pytest.mark.parametrize('kernel', kernels) @@ -66,7 +90,7 @@ def test_lml_precomputed(kernel): # Test that lml of optimized kernel is stored correctly. gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) assert (gpr.log_marginal_likelihood(gpr.kernel_.theta) == - gpr.log_marginal_likelihood()) + gpr.log_marginal_likelihood()) @pytest.mark.parametrize('kernel', kernels) @@ -160,6 +184,9 @@ def test_no_optimizer(): @pytest.mark.parametrize('kernel', kernels) def test_predict_cov_vs_std(kernel): + if sys.maxsize <= 2 ** 32 and sys.version_info[:2] == (3, 6): + pytest.xfail("This test may fail on 32bit Py3.6") + # Test that predicted std.-dev. is consistent with cov's diagonal. gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) y_mean, y_cov = gpr.predict(X2, return_cov=True) @@ -179,7 +206,7 @@ def test_anisotropic_kernel(): kernel = RBF([1.0, 1.0]) gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) assert (np.exp(gpr.kernel_.theta[1]) > - np.exp(gpr.kernel_.theta[0]) * 5) + np.exp(gpr.kernel_.theta[0]) * 5) def test_random_starts(): @@ -207,33 +234,103 @@ def test_random_starts(): @pytest.mark.parametrize('kernel', kernels) def test_y_normalization(kernel): - # Test normalization of the target values in GP + """ + Test normalization of the target values in GP - # Fitting non-normalizing GP on normalized y and fitting normalizing GP - # on unnormalized y should yield identical results - y_mean = y.mean(0) - y_norm = y - y_mean + Fitting non-normalizing GP on normalized y and fitting normalizing GP + on unnormalized y should yield identical results. Note that, here, + 'normalized y' refers to y that has been made zero mean and unit + variance. + + """ + + y_mean = np.mean(y) + y_std = np.std(y) + y_norm = (y - y_mean) / y_std # Fit non-normalizing GP on normalized y gpr = GaussianProcessRegressor(kernel=kernel) gpr.fit(X, y_norm) + # Fit normalizing GP on unnormalized y gpr_norm = GaussianProcessRegressor(kernel=kernel, normalize_y=True) gpr_norm.fit(X, y) # Compare predicted mean, std-devs and covariances y_pred, y_pred_std = gpr.predict(X2, return_std=True) - y_pred = y_mean + y_pred + y_pred = y_pred * y_std + y_mean + y_pred_std = y_pred_std * y_std y_pred_norm, y_pred_std_norm = gpr_norm.predict(X2, return_std=True) assert_almost_equal(y_pred, y_pred_norm) assert_almost_equal(y_pred_std, y_pred_std_norm) _, y_cov = gpr.predict(X2, return_cov=True) + y_cov = y_cov * y_std**2 _, y_cov_norm = gpr_norm.predict(X2, return_cov=True) + assert_almost_equal(y_cov, y_cov_norm) +def test_large_variance_y(): + """ + Here we test that, when noramlize_y=True, our GP can produce a + sensible fit to training data whose variance is significantly + larger than unity. This test was made in response to issue #15612. + + GP predictions are verified against predictions that were made + using GPy which, here, is treated as the 'gold standard'. Note that we + only investigate the RBF kernel here, as that is what was used in the + GPy implementation. + + The following code can be used to recreate the GPy data: + + -------------------------------------------------------------------------- + import GPy + + kernel_gpy = GPy.kern.RBF(input_dim=1, lengthscale=1.) + gpy = GPy.models.GPRegression(X, np.vstack(y_large), kernel_gpy) + gpy.optimize() + y_pred_gpy, y_var_gpy = gpy.predict(X2) + y_pred_std_gpy = np.sqrt(y_var_gpy) + -------------------------------------------------------------------------- + """ + + # Here we utilise a larger variance version of the training data + y_large = 10 * y + + # Standard GP with normalize_y=True + RBF_params = {'length_scale': 1.0} + kernel = RBF(**RBF_params) + gpr = GaussianProcessRegressor(kernel=kernel, normalize_y=True) + gpr.fit(X, y_large) + y_pred, y_pred_std = gpr.predict(X2, return_std=True) + + # 'Gold standard' mean predictions from GPy + y_pred_gpy = np.array([15.16918303, + -27.98707845, + -39.31636019, + 14.52605515, + 69.18503589]) + + # 'Gold standard' std predictions from GPy + y_pred_std_gpy = np.array([7.78860962, + 3.83179178, + 0.63149951, + 0.52745188, + 0.86170042]) + + # Based on numerical experiments, it's reasonable to expect our + # GP's mean predictions to get within 7% of predictions of those + # made by GPy. + assert_allclose(y_pred, y_pred_gpy, rtol=0.07, atol=0) + + # Based on numerical experiments, it's reasonable to expect our + # GP's std predictions to get within 15% of predictions of those + # made by GPy. + assert_allclose(y_pred_std, y_pred_std_gpy, rtol=0.15, atol=0) + + def test_y_multioutput(): # Test that GPR can deal with multi-dimensional target values y_2d = np.vstack((y, y * 2)).T @@ -297,7 +394,7 @@ def optimizer(obj_func, initial_theta, bounds): gpr.fit(X, y) # Checks that optimizer improved marginal likelihood assert (gpr.log_marginal_likelihood(gpr.kernel_.theta) > - gpr.log_marginal_likelihood(gpr.kernel.theta)) + gpr.log_marginal_likelihood(gpr.kernel.theta)) def test_gpr_correct_error_message(): diff --git a/sklearn/gaussian_process/tests/test_kernels.py b/sklearn/gaussian_process/tests/test_kernels.py index ecb636d13103b..9e2248a66ee28 100644 --- a/sklearn/gaussian_process/tests/test_kernels.py +++ b/sklearn/gaussian_process/tests/test_kernels.py @@ -14,22 +14,23 @@ from sklearn.gaussian_process.kernels \ import (RBF, Matern, RationalQuadratic, ExpSineSquared, DotProduct, ConstantKernel, WhiteKernel, PairwiseKernel, KernelOperator, - Exponentiation, Kernel) + Exponentiation, Kernel, CompoundKernel) from sklearn.base import clone from sklearn.utils._testing import (assert_almost_equal, assert_array_equal, - assert_array_almost_equal, - assert_raise_message) + assert_array_almost_equal, + assert_allclose, + assert_raise_message) X = np.random.RandomState(0).normal(0, 1, (5, 2)) Y = np.random.RandomState(0).normal(0, 1, (6, 2)) -kernel_white = RBF(length_scale=2.0) + WhiteKernel(noise_level=3.0) +kernel_rbf_plus_white = RBF(length_scale=2.0) + WhiteKernel(noise_level=3.0) kernels = [RBF(length_scale=2.0), RBF(length_scale_bounds=(0.5, 2.0)), ConstantKernel(constant_value=10.0), 2.0 * RBF(length_scale=0.33, length_scale_bounds="fixed"), - 2.0 * RBF(length_scale=0.5), kernel_white, + 2.0 * RBF(length_scale=0.5), kernel_rbf_plus_white, 2.0 * RBF(length_scale=[0.5, 2.0]), 2.0 * Matern(length_scale=0.33, length_scale_bounds="fixed"), 2.0 * Matern(length_scale=0.5, nu=0.5), @@ -92,8 +93,7 @@ def test_kernel_theta(kernel): # Check that values returned in theta are consistent with # hyperparameter values (being their logarithms) for i, hyperparameter in enumerate(kernel.hyperparameters): - assert (theta[i] == - np.log(getattr(kernel, hyperparameter.name))) + assert (theta[i] == np.log(getattr(kernel, hyperparameter.name))) # Fixed kernel parameters must be excluded from theta and gradient. for i, hyperparameter in enumerate(kernel.hyperparameters): @@ -129,7 +129,7 @@ def test_kernel_theta(kernel): @pytest.mark.parametrize('kernel', [kernel for kernel in kernels # Identity is not satisfied on diagonal - if kernel != kernel_white]) + if kernel != kernel_rbf_plus_white]) def test_auto_vs_cross(kernel): # Auto-correlation and cross-correlation should be consistent. K_auto = kernel(X) @@ -186,6 +186,27 @@ def test_kernel_stationary(kernel): assert_almost_equal(K[0, 0], np.diag(K)) +@pytest.mark.parametrize('kernel', kernels) +def test_kernel_input_type(kernel): + # Test whether kernels is for vectors or structured data + if isinstance(kernel, Exponentiation): + assert(kernel.requires_vector_input == + kernel.kernel.requires_vector_input) + if isinstance(kernel, KernelOperator): + assert(kernel.requires_vector_input == + (kernel.k1.requires_vector_input or + kernel.k2.requires_vector_input)) + + +def test_compound_kernel_input_type(): + kernel = CompoundKernel([WhiteKernel(noise_level=3.0)]) + assert not kernel.requires_vector_input + + kernel = CompoundKernel([WhiteKernel(noise_level=3.0), + RBF(length_scale=2.0)]) + assert kernel.requires_vector_input + + def check_hyperparameters_equal(kernel1, kernel2): # Check that hyperparameters of two kernels are equal for attr in set(dir(kernel1) + dir(kernel2)): @@ -229,6 +250,7 @@ def test_kernel_clone_after_set_params(kernel): isotropic_kernels): length_scale = params['length_scale'] if np.iterable(length_scale): + # XXX unreached code as of v0.22 params['length_scale'] = length_scale[0] params['length_scale_bounds'] = bounds else: @@ -236,8 +258,7 @@ def test_kernel_clone_after_set_params(kernel): params['length_scale_bounds'] = bounds * 2 kernel_cloned.set_params(**params) kernel_cloned_clone = clone(kernel_cloned) - assert (kernel_cloned_clone.get_params() == - kernel_cloned.get_params()) + assert (kernel_cloned_clone.get_params() == kernel_cloned.get_params()) assert id(kernel_cloned_clone) != id(kernel_cloned) check_hyperparameters_equal(kernel_cloned, kernel_cloned_clone) @@ -251,6 +272,11 @@ def test_matern_kernel(): K_absexp = np.exp(-euclidean_distances(X, X, squared=False)) K = Matern(nu=0.5, length_scale=1.0)(X) assert_array_almost_equal(K, K_absexp) + # matern kernel with coef0==inf is equal to RBF kernel + K_rbf = RBF(length_scale=1.0)(X) + K = Matern(nu=np.inf, length_scale=1.0)(X) + assert_array_almost_equal(K, K_rbf) + assert_allclose(K, K_rbf) # test that special cases of matern kernel (coef0 in [0.5, 1.5, 2.5]) # result in nearly identical results as the general case for coef0 in # [0.5 + tiny, 1.5 + tiny, 2.5 + tiny] @@ -259,6 +285,11 @@ def test_matern_kernel(): K1 = Matern(nu=nu, length_scale=1.0)(X) K2 = Matern(nu=nu + tiny, length_scale=1.0)(X) assert_array_almost_equal(K1, K2) + # test that coef0==large is close to RBF + large = 100 + K1 = Matern(nu=large, length_scale=1.0)(X) + K2 = RBF(length_scale=1.0)(X) + assert_array_almost_equal(K1, K2, decimal=2) @pytest.mark.parametrize("kernel", kernels) @@ -266,7 +297,7 @@ def test_kernel_versus_pairwise(kernel): # Check that GP kernels can also be used as pairwise kernels. # Test auto-kernel - if kernel != kernel_white: + if kernel != kernel_rbf_plus_white: # For WhiteKernel: k(X) != k(X,X). This is assumed by # pairwise_kernels K1 = kernel(X) diff --git a/sklearn/impute/_base.py b/sklearn/impute/_base.py index 2ad49833641dc..bc98778d5c5d8 100644 --- a/sklearn/impute/_base.py +++ b/sklearn/impute/_base.py @@ -120,13 +120,17 @@ class SimpleImputer(_BaseImputer): Read more in the :ref:`User Guide `. + .. versionadded:: 0.20 + `SimpleImputer` replaces the previous `sklearn.preprocessing.Imputer` + estimator which is now removed. + Parameters ---------- missing_values : number, string, np.nan (default) or None The placeholder for the missing values. All occurrences of `missing_values` will be imputed. - strategy : string, optional (default="mean") + strategy : string, default='mean' The imputation strategy. - If "mean", then replace missing values using the mean along @@ -141,16 +145,16 @@ class SimpleImputer(_BaseImputer): .. versionadded:: 0.20 strategy="constant" for fixed value imputation. - fill_value : string or numerical value, optional (default=None) + fill_value : string or numerical value, default=None When strategy == "constant", fill_value is used to replace all occurrences of missing_values. If left to the default, fill_value will be 0 when imputing numerical data and "missing_value" for strings or object data types. - verbose : integer, optional (default=0) + verbose : integer, default=0 Controls the verbosity of the imputer. - copy : boolean, optional (default=True) + copy : boolean, default=True If True, a copy of X will be created. If False, imputation will be done in-place whenever possible. Note that, in the following cases, a new copy will always be made, even if `copy=False`: @@ -159,7 +163,7 @@ class SimpleImputer(_BaseImputer): - If X is encoded as a CSR matrix; - If add_indicator=True. - add_indicator : boolean, optional (default=False) + add_indicator : boolean, default=False If True, a :class:`MissingIndicator` transform will stack onto output of the imputer's transform. This allows a predictive estimator to account for missingness despite imputation. If a feature has no @@ -213,7 +217,7 @@ def __init__(self, missing_values=np.nan, strategy="mean", self.verbose = verbose self.copy = copy - def _validate_input(self, X): + def _validate_input(self, X, in_fit): allowed_strategies = ["mean", "median", "most_frequent", "constant"] if self.strategy not in allowed_strategies: raise ValueError("Can only use these strategies: {0} " @@ -231,8 +235,10 @@ def _validate_input(self, X): force_all_finite = "allow-nan" try: - X = check_array(X, accept_sparse='csc', dtype=dtype, - force_all_finite=force_all_finite, copy=self.copy) + X = self._validate_data(X, reset=in_fit, + accept_sparse='csc', dtype=dtype, + force_all_finite=force_all_finite, + copy=self.copy) except ValueError as ve: if "could not convert" in str(ve): new_ve = ValueError("Cannot use {} strategy with non-numeric " @@ -265,7 +271,7 @@ def fit(self, X, y=None): ------- self : SimpleImputer """ - X = self._validate_input(X) + X = self._validate_input(X, in_fit=True) super()._fit_indicator(X) # default fill_value is 0 for numerical input and "missing_value" @@ -403,7 +409,7 @@ def transform(self, X): """ check_is_fitted(self) - X = self._validate_input(X) + X = self._validate_input(X, in_fit=False) X_indicator = super()._transform_indicator(X) statistics = self.statistics_ @@ -463,6 +469,8 @@ class MissingIndicator(TransformerMixin, BaseEstimator): Read more in the :ref:`User Guide `. + .. versionadded:: 0.20 + Parameters ---------- missing_values : number, string, np.nan (default) or None @@ -470,7 +478,7 @@ class MissingIndicator(TransformerMixin, BaseEstimator): `missing_values` will be indicated (True in the output array), the other values will be marked as False. - features : str, optional + features : str, default=None Whether the imputer mask should represent all or a subset of features. @@ -478,7 +486,7 @@ class MissingIndicator(TransformerMixin, BaseEstimator): features containing missing values during fit time. - If "all", the imputer mask will represent all features. - sparse : boolean or "auto", optional + sparse : boolean or "auto", default=None Whether the imputer mask format should be sparse or dense. - If "auto" (default), the imputer mask will be of same type as @@ -486,7 +494,7 @@ class MissingIndicator(TransformerMixin, BaseEstimator): - If True, the imputer mask will be a sparse matrix. - If False, the imputer mask will be a numpy array. - error_on_new : boolean, optional + error_on_new : boolean, default=None If True (default), transform will raise an error when there are features with missing values in transform that have no missing values in fit. This is applicable only when ``features="missing-only"``. @@ -581,13 +589,14 @@ def _get_missing_features_info(self, X): return imputer_mask, features_indices - def _validate_input(self, X): + def _validate_input(self, X, in_fit): if not is_scalar_nan(self.missing_values): force_all_finite = True else: force_all_finite = "allow-nan" - X = check_array(X, accept_sparse=('csc', 'csr'), dtype=None, - force_all_finite=force_all_finite) + X = self._validate_data(X, reset=in_fit, + accept_sparse=('csc', 'csr'), dtype=None, + force_all_finite=force_all_finite) _check_inputs_dtype(X, self.missing_values) if X.dtype.kind not in ("i", "u", "f", "O"): raise ValueError("MissingIndicator does not support data with " @@ -622,7 +631,7 @@ def _fit(self, X, y=None): The imputer mask of the original data. """ - X = self._validate_input(X) + X = self._validate_input(X, in_fit=True) self._n_features = X.shape[1] if self.features not in ('missing-only', 'all'): @@ -674,7 +683,7 @@ def transform(self, X): """ check_is_fitted(self) - X = self._validate_input(X) + X = self._validate_input(X, in_fit=False) if X.shape[1] != self._n_features: raise ValueError("X has a different number of features " diff --git a/sklearn/impute/_iterative.py b/sklearn/impute/_iterative.py index fa9d576f04008..58a35d157c7a4 100644 --- a/sklearn/impute/_iterative.py +++ b/sklearn/impute/_iterative.py @@ -11,9 +11,9 @@ from ..base import clone from ..exceptions import ConvergenceWarning from ..preprocessing import normalize -from ..utils import check_array, check_random_state, _safe_indexing +from ..utils import (check_array, check_random_state, _safe_indexing, + is_scalar_nan) from ..utils.validation import FLOAT_DTYPES, check_is_fitted -from ..utils import is_scalar_nan from ..utils._mask import _get_mask from ._base import _BaseImputer @@ -34,6 +34,8 @@ class IterativeImputer(_BaseImputer): Read more in the :ref:`User Guide `. + .. versionadded:: 0.21 + .. note:: This estimator is still **experimental** for now: the predictions @@ -52,7 +54,7 @@ class IterativeImputer(_BaseImputer): If ``sample_posterior`` is True, the estimator must support ``return_std`` in its ``predict`` method. - missing_values : int, np.nan, optional (default=np.nan) + missing_values : int, np.nan, default=np.nan The placeholder for the missing values. All occurrences of ``missing_values`` will be imputed. @@ -62,7 +64,7 @@ class IterativeImputer(_BaseImputer): ``return_std`` in its ``predict`` method if set to ``True``. Set to ``True`` if using ``IterativeImputer`` for multiple imputations. - max_iter : int, optional (default=10) + max_iter : int, default=10 Maximum number of imputation rounds to perform before returning the imputations computed during the final round. A round is a single imputation of each feature with missing values. The stopping criterion @@ -70,10 +72,10 @@ class IterativeImputer(_BaseImputer): where `X_t` is `X` at iteration `t. Note that early stopping is only applied if ``sample_posterior=False``. - tol : float, optional (default=1e-3) + tol : float, default=1e-3 Tolerance of the stopping condition. - n_nearest_features : int, optional (default=None) + n_nearest_features : int, default=None Number of other features to use to estimate the missing values of each feature column. Nearness between features is measured using the absolute correlation coefficient between each feature pair (after @@ -83,12 +85,12 @@ class IterativeImputer(_BaseImputer): imputed target feature. Can provide significant speed-up when the number of features is huge. If ``None``, all features will be used. - initial_strategy : str, optional (default="mean") + initial_strategy : str, default='mean' Which strategy to use to initialize the missing values. Same as the ``strategy`` parameter in :class:`sklearn.impute.SimpleImputer` Valid values: {"mean", "median", "most_frequent", or "constant"}. - imputation_order : str, optional (default="ascending") + imputation_order : str, default='ascending' The order in which the features will be imputed. Possible values: "ascending" @@ -102,34 +104,36 @@ class IterativeImputer(_BaseImputer): "random" A random order for each round. - skip_complete : boolean, optional (default=False) + skip_complete : boolean, default=False If ``True`` then features with missing values during ``transform`` which did not have any missing values during ``fit`` will be imputed with the initial imputation method only. Set to ``True`` if you have many features with no missing values at both ``fit`` and ``transform`` time to save compute. - min_value : float, optional (default=None) - Minimum possible imputed value. Default of ``None`` will set minimum - to negative infinity. + min_value : float or array-like of shape (n_features,), default=None. + Minimum possible imputed value. Broadcast to shape (n_features,) if + scalar. If array-like, expects shape (n_features,), one min value for + each feature. `None` (default) is converted to -np.inf. - max_value : float, optional (default=None) - Maximum possible imputed value. Default of ``None`` will set maximum - to positive infinity. + max_value : float or array-like of shape (n_features,), default=None. + Maximum possible imputed value. Broadcast to shape (n_features,) if + scalar. If array-like, expects shape (n_features,), one max value for + each feature. `None` (default) is converted to np.inf. - verbose : int, optional (default=0) + verbose : int, default=0 Verbosity flag, controls the debug messages that are issued as functions are evaluated. The higher, the more verbose. Can be 0, 1, or 2. - random_state : int, RandomState instance or None, optional (default=None) + random_state : int, RandomState instance or None, default=None The seed of the pseudo random number generator to use. Randomizes selection of estimator features if n_nearest_features is not None, the ``imputation_order`` if ``random``, and the sampling from posterior if ``sample_posterior`` is True. Use an integer for determinism. See :term:`the Glossary `. - add_indicator : boolean, optional (default=False) + add_indicator : boolean, default=False If True, a :class:`MissingIndicator` transform will stack onto output of the imputer's transform. This allows a predictive estimator to account for missingness despite imputation. If a feature has no @@ -172,7 +176,7 @@ class IterativeImputer(_BaseImputer): Examples -------- >>> import numpy as np - >>> from sklearn.experimental import enable_iterative_imputer + >>> from sklearn.experimental import enable_iterative_imputer >>> from sklearn.impute import IterativeImputer >>> imp_mean = IterativeImputer(random_state=0) >>> imp_mean.fit([[7, 2, 3], [4, np.nan, 6], [10, 5, 9]]) @@ -294,9 +298,9 @@ def _impute_one_feature(self, missing_row_mask = mask_missing_values[:, feat_idx] if fit_mode: X_train = _safe_indexing(X_filled[:, neighbor_feat_idx], - ~missing_row_mask) + ~missing_row_mask) y_train = _safe_indexing(X_filled[:, feat_idx], - ~missing_row_mask) + ~missing_row_mask) estimator.fit(X_train, y_train) # if no missing values, don't predict @@ -314,16 +318,16 @@ def _impute_one_feature(self, # (results in inf sample) positive_sigmas = sigmas > 0 imputed_values[~positive_sigmas] = mus[~positive_sigmas] - mus_too_low = mus < self._min_value - imputed_values[mus_too_low] = self._min_value - mus_too_high = mus > self._max_value - imputed_values[mus_too_high] = self._max_value + mus_too_low = mus < self._min_value[feat_idx] + imputed_values[mus_too_low] = self._min_value[feat_idx] + mus_too_high = mus > self._max_value[feat_idx] + imputed_values[mus_too_high] = self._max_value[feat_idx] # the rest can be sampled without statistical issues inrange_mask = positive_sigmas & ~mus_too_low & ~mus_too_high mus = mus[inrange_mask] sigmas = sigmas[inrange_mask] - a = (self._min_value - mus) / sigmas - b = (self._max_value - mus) / sigmas + a = (self._min_value[feat_idx] - mus) / sigmas + b = (self._max_value[feat_idx] - mus) / sigmas if scipy.__version__ < LooseVersion('0.18'): # bug with vector-valued `a` in old scipy @@ -341,8 +345,8 @@ def _impute_one_feature(self, else: imputed_values = estimator.predict(X_test) imputed_values = np.clip(imputed_values, - self._min_value, - self._max_value) + self._min_value[feat_idx], + self._max_value[feat_idx]) # update the feature X_filled[missing_row_mask, feat_idx] = imputed_values @@ -443,7 +447,7 @@ def _get_abs_corr_mat(self, X_filled, tolerance=1e-6): X_filled : ndarray, shape (n_samples, n_features) Input data with the most recent imputations. - tolerance : float, optional (default=1e-6) + tolerance : float, default=1e-6 ``abs_corr_mat`` can have nans, which will be replaced with ``tolerance``. @@ -501,8 +505,8 @@ def _initial_imputation(self, X): else: force_all_finite = True - X = check_array(X, dtype=FLOAT_DTYPES, order="F", - force_all_finite=force_all_finite) + X = self._validate_data(X, dtype=FLOAT_DTYPES, order="F", + force_all_finite=force_all_finite) _check_inputs_dtype(X, self.missing_values) mask_missing_values = _get_mask(X, self.missing_values) @@ -522,6 +526,38 @@ def _initial_imputation(self, X): return Xt, X_filled, mask_missing_values + @staticmethod + def _validate_limit(limit, limit_type, n_features): + """Validate the limits (min/max) of the feature values + Converts scalar min/max limits to vectors of shape (n_features,) + + Parameters + ---------- + limit: scalar or array-like + The user-specified limit (i.e, min_value or max_value) + limit_type: string, "max" or "min" + n_features: Number of features in the dataset + + Returns + ------- + limit: ndarray, shape(n_features,) + Array of limits, one for each feature + """ + limit_bound = np.inf if limit_type == "max" else -np.inf + limit = limit_bound if limit is None else limit + if np.isscalar(limit): + limit = np.full(n_features, limit) + limit = check_array( + limit, force_all_finite=False, copy=False, ensure_2d=False + ) + if not limit.shape[0] == n_features: + raise ValueError( + f"'{limit_type}_value' should be of " + f"shape ({n_features},) when an array-like " + f"is provided. Got {limit.shape}, instead." + ) + return limit + def fit_transform(self, X, y=None): """Fits the imputer on X and return the transformed X. @@ -563,9 +599,6 @@ def fit_transform(self, X, y=None): self.imputation_sequence_ = [] - self._min_value = -np.inf if self.min_value is None else self.min_value - self._max_value = np.inf if self.max_value is None else self.max_value - self.initial_imputer_ = None super()._fit_indicator(X) X_indicator = super()._transform_indicator(X) @@ -579,6 +612,15 @@ def fit_transform(self, X, y=None): self.n_iter_ = 0 return super()._concatenate_indicator(Xt, X_indicator) + self._min_value = IterativeImputer._validate_limit( + self.min_value, "min", X.shape[1]) + self._max_value = IterativeImputer._validate_limit( + self.max_value, "max", X.shape[1]) + + if not np.all(np.greater(self._max_value, self._min_value)): + raise ValueError( + "One (or more) features have min_value >= max_value.") + # order in which to impute # note this is probably too slow for large feature data (d > 100000) # and a better way would be good. diff --git a/sklearn/impute/_knn.py b/sklearn/impute/_knn.py index fe1a94304ed57..f782a46a6b40d 100644 --- a/sklearn/impute/_knn.py +++ b/sklearn/impute/_knn.py @@ -6,7 +6,7 @@ from ._base import _BaseImputer from ..utils.validation import FLOAT_DTYPES -from ..metrics import pairwise_distances +from ..metrics import pairwise_distances_chunked from ..metrics.pairwise import _NAN_METRICS from ..neighbors._base import _get_weights from ..neighbors._base import _check_weights @@ -178,8 +178,9 @@ def fit(self, X, y=None): raise ValueError( "Expected n_neighbors > 0. Got {}".format(self.n_neighbors)) - X = check_array(X, accept_sparse=False, dtype=FLOAT_DTYPES, - force_all_finite=force_all_finite, copy=self.copy) + X = self._validate_data(X, accept_sparse=False, dtype=FLOAT_DTYPES, + force_all_finite=force_all_finite, + copy=self.copy) super()._fit_indicator(X) _check_weights(self.weights) @@ -217,71 +218,81 @@ def transform(self, X): mask = _get_mask(X, self.missing_values) mask_fit_X = self._mask_fit_X + valid_mask = ~np.all(mask_fit_X, axis=0) - # Removes columns where the training data is all nan if not np.any(mask): - valid_mask = ~np.all(mask_fit_X, axis=0) + # No missing values in X + # Remove columns where the training data is all nan return X[:, valid_mask] row_missing_idx = np.flatnonzero(mask.any(axis=1)) - # Pairwise distances between receivers and fitted samples - dist = pairwise_distances(X[row_missing_idx, :], self._fit_X, - metric=self.metric, - missing_values=self.missing_values, - force_all_finite=force_all_finite) + non_missing_fix_X = np.logical_not(mask_fit_X) # Maps from indices from X to indices in dist matrix dist_idx_map = np.zeros(X.shape[0], dtype=np.int) dist_idx_map[row_missing_idx] = np.arange(row_missing_idx.shape[0]) - non_missing_fix_X = np.logical_not(mask_fit_X) - - # Find and impute missing - valid_idx = [] - for col in range(X.shape[1]): - - potential_donors_idx = np.flatnonzero(non_missing_fix_X[:, col]) - - # column was all missing during training - if len(potential_donors_idx) == 0: - continue + def process_chunk(dist_chunk, start): + row_missing_chunk = row_missing_idx[start:start + len(dist_chunk)] - # column has no missing values - if not np.any(mask[:, col]): - valid_idx.append(col) - continue - - valid_idx.append(col) - - receivers_idx = np.flatnonzero(mask[:, col]) - - # distances for samples that needed imputation for column - dist_subset = (dist[dist_idx_map[receivers_idx]] - [:, potential_donors_idx]) + # Find and impute missing by column + for col in range(X.shape[1]): + if not valid_mask[col]: + # column was all missing during training + continue - # receivers with all nan distances impute with mean - all_nan_dist_mask = np.isnan(dist_subset).all(axis=1) - all_nan_receivers_idx = receivers_idx[all_nan_dist_mask] + col_mask = mask[row_missing_chunk, col] + if not np.any(col_mask): + # column has no missing values + continue - if all_nan_receivers_idx.size: - col_mean = np.ma.array(self._fit_X[:, col], - mask=mask_fit_X[:, col]).mean() - X[all_nan_receivers_idx, col] = col_mean + potential_donors_idx, = np.nonzero(non_missing_fix_X[:, col]) - if len(all_nan_receivers_idx) == len(receivers_idx): - # all receivers imputed with mean - continue + # receivers_idx are indices in X + receivers_idx = row_missing_chunk[np.flatnonzero(col_mask)] - # receivers with at least one defined distance - receivers_idx = receivers_idx[~all_nan_dist_mask] - dist_subset = (dist[dist_idx_map[receivers_idx]] + # distances for samples that needed imputation for column + dist_subset = (dist_chunk[dist_idx_map[receivers_idx] - start] [:, potential_donors_idx]) - n_neighbors = min(self.n_neighbors, len(potential_donors_idx)) - value = self._calc_impute(dist_subset, n_neighbors, - self._fit_X[potential_donors_idx, col], - mask_fit_X[potential_donors_idx, col]) - X[receivers_idx, col] = value - - return super()._concatenate_indicator(X[:, valid_idx], X_indicator) + # receivers with all nan distances impute with mean + all_nan_dist_mask = np.isnan(dist_subset).all(axis=1) + all_nan_receivers_idx = receivers_idx[all_nan_dist_mask] + + if all_nan_receivers_idx.size: + col_mean = np.ma.array(self._fit_X[:, col], + mask=mask_fit_X[:, col]).mean() + X[all_nan_receivers_idx, col] = col_mean + + if len(all_nan_receivers_idx) == len(receivers_idx): + # all receivers imputed with mean + continue + + # receivers with at least one defined distance + receivers_idx = receivers_idx[~all_nan_dist_mask] + dist_subset = (dist_chunk[dist_idx_map[receivers_idx] + - start] + [:, potential_donors_idx]) + + n_neighbors = min(self.n_neighbors, len(potential_donors_idx)) + value = self._calc_impute( + dist_subset, + n_neighbors, + self._fit_X[potential_donors_idx, col], + mask_fit_X[potential_donors_idx, col]) + X[receivers_idx, col] = value + + # process in fixed-memory chunks + gen = pairwise_distances_chunked( + X[row_missing_idx, :], + self._fit_X, + metric=self.metric, + missing_values=self.missing_values, + force_all_finite=force_all_finite, + reduce_func=process_chunk) + for chunk in gen: + # process_chunk modifies X in place. No return value. + pass + + return super()._concatenate_indicator(X[:, valid_mask], X_indicator) diff --git a/sklearn/impute/tests/test_impute.py b/sklearn/impute/tests/test_impute.py index 052a99908c569..50f60ff6e96ad 100644 --- a/sklearn/impute/tests/test_impute.py +++ b/sklearn/impute/tests/test_impute.py @@ -177,6 +177,7 @@ def test_imputation_mean_median(): X[:, j] = np.hstack((v, z, p)) if 0 == test_missing_values: + # XXX unreached code as of v0.22 X_true[:, j] = np.hstack((v, np.repeat( true_statistics[j], @@ -706,7 +707,6 @@ def test_iterative_imputer_truncated_normal_posterior(): # note that starting from the wrong random seed will make this test fail # because random sampling doesn't occur at all when the imputation # is outside of the (min_value, max_value) range - pytest.importorskip("scipy", minversion="0.17.0") rng = np.random.RandomState(42) X = rng.normal(size=(5, 5)) @@ -763,7 +763,6 @@ def test_iterative_imputer_missing_at_transform(strategy): def test_iterative_imputer_transform_stochasticity(): - pytest.importorskip("scipy", minversion="0.17.0") rng1 = np.random.RandomState(0) rng2 = np.random.RandomState(1) n = 100 @@ -971,6 +970,74 @@ def test_iterative_imputer_catch_warning(): assert not np.any(np.isnan(X_fill)) +@pytest.mark.parametrize( + "min_value, max_value, correct_output", + [(0, 100, np.array([[0] * 3, [100] * 3])), + (None, None, np.array([[-np.inf] * 3, [np.inf] * 3])), + (-np.inf, np.inf, np.array([[-np.inf] * 3, [np.inf] * 3])), + ([-5, 5, 10], [100, 200, 300], np.array([[-5, 5, 10], [100, 200, 300]])), + ([-5, -np.inf, 10], [100, 200, np.inf], + np.array([[-5, -np.inf, 10], [100, 200, np.inf]]))], + ids=["scalars", "None-default", "inf", "lists", "lists-with-inf"]) +def test_iterative_imputer_min_max_array_like(min_value, + max_value, + correct_output): + # check that passing scalar or array-like + # for min_value and max_value in IterativeImputer works + X = np.random.RandomState(0).randn(10, 3) + imputer = IterativeImputer(min_value=min_value, max_value=max_value) + imputer.fit(X) + + assert (isinstance(imputer._min_value, np.ndarray) and + isinstance(imputer._max_value, np.ndarray)) + assert ((imputer._min_value.shape[0] == X.shape[1]) and + (imputer._max_value.shape[0] == X.shape[1])) + + assert_allclose(correct_output[0, :], imputer._min_value) + assert_allclose(correct_output[1, :], imputer._max_value) + + +@pytest.mark.parametrize( + "min_value, max_value, err_msg", + [(100, 0, "min_value >= max_value."), + (np.inf, -np.inf, "min_value >= max_value."), + ([-5, 5], [100, 200, 0], "_value' should be of shape")]) +def test_iterative_imputer_catch_min_max_error(min_value, max_value, err_msg): + # check that passing scalar or array-like + # for min_value and max_value in IterativeImputer works + X = np.random.random((10, 3)) + imputer = IterativeImputer(min_value=min_value, max_value=max_value) + with pytest.raises(ValueError, match=err_msg): + imputer.fit(X) + + +@pytest.mark.parametrize( + "min_max_1, min_max_2", + [([None, None], [-np.inf, np.inf]), + ([-10, 10], [[-10] * 4, [10] * 4])], + ids=["None-vs-inf", "Scalar-vs-vector"]) +def test_iterative_imputer_min_max_array_like_imputation(min_max_1, min_max_2): + # Test that None/inf and scalar/vector give the same imputation + X_train = np.array([ + [np.nan, 2, 2, 1], + [10, np.nan, np.nan, 7], + [3, 1, np.nan, 1], + [np.nan, 4, 2, np.nan]]) + X_test = np.array([ + [np.nan, 2, np.nan, 5], + [2, 4, np.nan, np.nan], + [np.nan, 1, 10, 1]]) + imputer1 = IterativeImputer(min_value=min_max_1[0], + max_value=min_max_1[1], + random_state=0) + imputer2 = IterativeImputer(min_value=min_max_2[0], + max_value=min_max_2[1], + random_state=0) + X_test_imputed1 = imputer1.fit(X_train).transform(X_test) + X_test_imputed2 = imputer2.fit(X_train).transform(X_test) + assert_allclose(X_test_imputed1[:, 0], X_test_imputed2[:, 0]) + + @pytest.mark.parametrize( "skip_complete", [True, False] ) diff --git a/sklearn/impute/tests/test_knn.py b/sklearn/impute/tests/test_knn.py index e9c89c03f89b8..68c4d9f3cc54a 100644 --- a/sklearn/impute/tests/test_knn.py +++ b/sklearn/impute/tests/test_knn.py @@ -1,22 +1,14 @@ import numpy as np import pytest +from sklearn import config_context from sklearn.impute import KNNImputer from sklearn.metrics.pairwise import nan_euclidean_distances from sklearn.metrics.pairwise import pairwise_distances from sklearn.neighbors import KNeighborsRegressor -from sklearn.utils._mask import _get_mask from sklearn.utils._testing import assert_allclose -def _missing_mean(X, missing_value): - masked_X = np.ma.array(X, mask=_get_mask(X, missing_value)) - masked_X_mean = masked_X.mean(axis=0) - output = masked_X_mean.data - output[masked_X_mean.mask] = np.nan - return output - - @pytest.mark.parametrize("weights", ["uniform", "distance"]) @pytest.mark.parametrize("n_neighbors", range(1, 6)) def test_knn_imputer_shape(weights, n_neighbors): @@ -522,8 +514,12 @@ def custom_callable(x, y, missing_values=np.nan, squared=False): assert_allclose(imputer.fit_transform(X), X_imputed) +@pytest.mark.parametrize("working_memory", [None, 0]) @pytest.mark.parametrize("na", [-1, np.nan]) -def test_knn_imputer_with_simple_example(na): +# Note that we use working_memory=0 to ensure that chunking is tested, even +# for a small dataset. However, it should raise a UserWarning that we ignore. +@pytest.mark.filterwarnings("ignore:adhere to working_memory") +def test_knn_imputer_with_simple_example(na, working_memory): X = np.array([ [0, na, 0, na], @@ -553,8 +549,9 @@ def test_knn_imputer_with_simple_example(na): [r7c0, 7, 7, 7] ]) - imputer_comp = KNNImputer(missing_values=na) - assert_allclose(imputer_comp.fit_transform(X), X_imputed) + with config_context(working_memory=working_memory): + imputer_comp = KNNImputer(missing_values=na) + assert_allclose(imputer_comp.fit_transform(X), X_imputed) @pytest.mark.parametrize("na", [-1, np.nan]) @@ -598,8 +595,10 @@ def test_knn_imputer_drops_all_nan_features(na): assert_allclose(knn.transform(X2), X2_expected) +@pytest.mark.parametrize("working_memory", [None, 0]) @pytest.mark.parametrize("na", [-1, np.nan]) -def test_knn_imputer_distance_weighted_not_enough_neighbors(na): +def test_knn_imputer_distance_weighted_not_enough_neighbors(na, + working_memory): X = np.array([ [3, na], [2, na], @@ -626,11 +625,14 @@ def test_knn_imputer_distance_weighted_not_enough_neighbors(na): [X_50, 5] ]) - knn_3 = KNNImputer(missing_values=na, n_neighbors=3, weights='distance') - assert_allclose(knn_3.fit_transform(X), X_expected) + with config_context(working_memory=working_memory): + knn_3 = KNNImputer(missing_values=na, n_neighbors=3, + weights='distance') + assert_allclose(knn_3.fit_transform(X), X_expected) - knn_4 = KNNImputer(missing_values=na, n_neighbors=4, weights='distance') - assert_allclose(knn_4.fit_transform(X), X_expected) + knn_4 = KNNImputer(missing_values=na, n_neighbors=4, + weights='distance') + assert_allclose(knn_4.fit_transform(X), X_expected) @pytest.mark.parametrize("na, allow_nan", [(-1, False), (np.nan, True)]) diff --git a/sklearn/inspection/__init__.py b/sklearn/inspection/__init__.py index 04d9d84ecaf02..5940ac22a2ef2 100644 --- a/sklearn/inspection/__init__.py +++ b/sklearn/inspection/__init__.py @@ -1,8 +1,22 @@ """The :mod:`sklearn.inspection` module includes tools for model inspection.""" -from ._partial_dependence import partial_dependence -from ._partial_dependence import plot_partial_dependence -from ._partial_dependence import PartialDependenceDisplay -from ._permutation_importance import permutation_importance + +# TODO: remove me in 0.24 (as well as the noqa markers) and +# import the partial_dependence func directly from the +# ._partial_dependence module instead. +# Pre-cache the import of the deprecated module so that import +# sklearn.inspection.partial_dependence returns the function as in +# 0.21, instead of the module +# https://github.com/scikit-learn/scikit-learn/issues/15842 +import warnings +with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=FutureWarning) + from .partial_dependence import partial_dependence + +from ._permutation_importance import permutation_importance # noqa + +from ._plot.partial_dependence import plot_partial_dependence # noqa +from ._plot.partial_dependence import PartialDependenceDisplay # noqa + __all__ = [ 'partial_dependence', diff --git a/sklearn/inspection/_partial_dependence.py b/sklearn/inspection/_partial_dependence.py index ca0b18b5b6173..f0fbc23333266 100644 --- a/sklearn/inspection/_partial_dependence.py +++ b/sklearn/inspection/_partial_dependence.py @@ -5,16 +5,11 @@ # Nicolas Hug # License: BSD 3 clause -from itertools import chain -from itertools import count -import numbers from collections.abc import Iterable -import warnings import numpy as np from scipy import sparse from scipy.stats.mstats import mquantiles -from joblib import Parallel, delayed from ..base import is_classifier, is_regressor from ..pipeline import Pipeline @@ -25,15 +20,17 @@ from ..utils import _determine_key_type from ..utils import _get_column_indices from ..utils.validation import check_is_fitted -from ..tree._tree import DTYPE +from ..tree import DecisionTreeRegressor +from ..ensemble import RandomForestRegressor from ..exceptions import NotFittedError from ..ensemble._gb import BaseGradientBoosting from sklearn.ensemble._hist_gradient_boosting.gradient_boosting import ( BaseHistGradientBoosting) -__all__ = ['partial_dependence', 'plot_partial_dependence', - 'PartialDependenceDisplay'] +__all__ = [ + 'partial_dependence', +] def _grid_from_X(X, percentiles, grid_resolution): @@ -105,7 +102,14 @@ def _grid_from_X(X, percentiles, grid_resolution): def _partial_dependence_recursion(est, grid, features): - return est._compute_partial_dependence_recursion(grid, features) + averaged_predictions = est._compute_partial_dependence_recursion(grid, + features) + if averaged_predictions.ndim == 1: + # reshape to (1, n_points) for consistency with + # _partial_dependence_brute + averaged_predictions = averaged_predictions.reshape(1, -1) + + return averaged_predictions def _partial_dependence_brute(est, grid, features, X, response_method): @@ -188,6 +192,23 @@ def partial_dependence(estimator, X, features, response_method='auto', Read more in the :ref:`User Guide `. + .. warning:: + + For :class:`~sklearn.ensemble.GradientBoostingClassifier` and + :class:`~sklearn.ensemble.GradientBoostingRegressor`, the + 'recursion' method (used by default) will not account for the `init` + predictor of the boosting process. In practice, this will produce + the same values as 'brute' up to a constant offset in the target + response, provided that `init` is a constant estimator (which is the + default). However, if `init` is not a constant estimator, the + partial dependence values are incorrect for 'recursion' because the + offset will be sample-dependent. It is preferable to use the 'brute' + method. Note that this only applies to + :class:`~sklearn.ensemble.GradientBoostingClassifier` and + :class:`~sklearn.ensemble.GradientBoostingRegressor`, not to + :class:`~sklearn.ensemble.HistGradientBoostingClassifier` and + :class:`~sklearn.ensemble.HistGradientBoostingRegressor`. + Parameters ---------- estimator : BaseEstimator @@ -196,9 +217,10 @@ def partial_dependence(estimator, X, features, response_method='auto', Multioutput-multiclass classifiers are not supported. X : {array-like or dataframe} of shape (n_samples, n_features) - ``X`` is used both to generate a grid of values for the - ``features``, and to compute the averaged predictions when - method is 'brute'. + ``X`` is used to generate a grid of values for the target + ``features`` (where the partial dependence will be evaluated), and + also to generate values for the complement features when the + `method` is 'brute'. features : array-like of {int, str} The feature (e.g. `[0]`) or pair of interacting features @@ -225,34 +247,27 @@ def partial_dependence(estimator, X, features, response_method='auto', method : str, optional (default='auto') The method used to calculate the averaged predictions: - - 'recursion' is only supported for gradient boosting estimator (namely - :class:`GradientBoostingClassifier`, - :class:`GradientBoostingRegressor`, - :class:`HistGradientBoostingClassifier`, - :class:`HistGradientBoostingRegressor`) + - 'recursion' is only supported for some tree-based estimators (namely + :class:`~sklearn.ensemble.GradientBoostingClassifier`, + :class:`~sklearn.ensemble.GradientBoostingRegressor`, + :class:`~sklearn.ensemble.HistGradientBoostingClassifier`, + :class:`~sklearn.ensemble.HistGradientBoostingRegressor`, + :class:`~sklearn.tree.DecisionTreeRegressor`, + :class:`~sklearn.ensemble.RandomForestRegressor`, + ) but is more efficient in terms of speed. - With this method, ``X`` is only used to build the - grid and the partial dependences are computed using the training - data. This method does not account for the ``init`` predictor of - the boosting process, which may lead to incorrect values (see - warning below). With this method, the target response of a + With this method, the target response of a classifier is always the decision function, not the predicted probabilities. - 'brute' is supported for any estimator, but is more computationally intensive. - - 'auto': + - 'auto': the 'recursion' is used for estimators that support it, + and 'brute' is used otherwise. - - 'recursion' is used for - :class:`GradientBoostingClassifier` - and - :class:`GradientBoostingRegressor` - if ``init=None``, and for - :class:`HistGradientBoostingClassifier` - and - :class:`HistGradientBoostingRegressor`. - - 'brute' is used for all other estimators. + Please see :ref:`this note ` for + differences between the 'brute' and 'recursion' method. Returns ------- @@ -286,21 +301,6 @@ def partial_dependence(estimator, X, features, response_method='auto', See also -------- sklearn.inspection.plot_partial_dependence: Plot partial dependence - - Warnings - -------- - The 'recursion' method only works for gradient boosting estimators, and - unlike the 'brute' method, it does not account for the ``init`` - predictor of the boosting process. In practice this will produce the - same values as 'brute' up to a constant offset in the target response, - provided that ``init`` is a consant estimator (which is the default). - However, as soon as ``init`` is not a constant estimator, the partial - dependence values are incorrect for 'recursion'. This is not relevant for - :class:`HistGradientBoostingClassifier - ` and - :class:`HistGradientBoostingRegressor - `, which do not have an - ``init`` parameter. """ if not (is_classifier(estimator) or is_regressor(estimator)): raise ValueError( @@ -351,19 +351,25 @@ def partial_dependence(estimator, X, features, response_method='auto', if (isinstance(estimator, BaseGradientBoosting) and estimator.init is None): method = 'recursion' - elif isinstance(estimator, BaseHistGradientBoosting): + elif isinstance(estimator, (BaseHistGradientBoosting, + DecisionTreeRegressor, + RandomForestRegressor)): method = 'recursion' else: method = 'brute' if method == 'recursion': if not isinstance(estimator, - (BaseGradientBoosting, BaseHistGradientBoosting)): + (BaseGradientBoosting, BaseHistGradientBoosting, + DecisionTreeRegressor, RandomForestRegressor)): supported_classes_recursion = ( 'GradientBoostingClassifier', 'GradientBoostingRegressor', 'HistGradientBoostingClassifier', 'HistGradientBoostingRegressor', + 'HistGradientBoostingRegressor', + 'DecisionTreeRegressor', + 'RandomForestRegressor', ) raise ValueError( "Only the following estimators support the 'recursion' " @@ -411,530 +417,3 @@ def partial_dependence(estimator, X, features, response_method='auto', -1, *[val.shape[0] for val in values]) return averaged_predictions, values - - -def plot_partial_dependence(estimator, X, features, feature_names=None, - target=None, response_method='auto', n_cols=3, - grid_resolution=100, percentiles=(0.05, 0.95), - method='auto', n_jobs=None, verbose=0, fig=None, - line_kw=None, contour_kw=None, ax=None): - """Partial dependence plots. - - The ``len(features)`` plots are arranged in a grid with ``n_cols`` - columns. Two-way partial dependence plots are plotted as contour plots. The - deciles of the feature values will be shown with tick marks on the x-axes - for one-way plots, and on both axes for two-way plots. - - Read more in the :ref:`User Guide `. - - Parameters - ---------- - estimator : BaseEstimator - A fitted estimator object implementing :term:`predict`, - :term:`predict_proba`, or :term:`decision_function`. - Multioutput-multiclass classifiers are not supported. - - X : {array-like or dataframe} of shape (n_samples, n_features) - The data to use to build the grid of values on which the dependence - will be evaluated. This is usually the training data. - - features : list of {int, str, pair of int, pair of str} - The target features for which to create the PDPs. - If features[i] is an int or a string, a one-way PDP is created; if - features[i] is a tuple, a two-way PDP is created. Each tuple must be - of size 2. - if any entry is a string, then it must be in ``feature_names``. - - feature_names : array-like of shape (n_features,), dtype=str, default=None - Name of each feature; feature_names[i] holds the name of the feature - with index i. - By default, the name of the feature corresponds to their numerical - index for NumPy array and their column name for pandas dataframe. - - target : int, optional (default=None) - - In a multiclass setting, specifies the class for which the PDPs - should be computed. Note that for binary classification, the - positive class (index 1) is always used. - - In a multioutput setting, specifies the task for which the PDPs - should be computed. - - Ignored in binary classification or classical regression settings. - - response_method : 'auto', 'predict_proba' or 'decision_function', \ - optional (default='auto') - Specifies whether to use :term:`predict_proba` or - :term:`decision_function` as the target response. For regressors - this parameter is ignored and the response is always the output of - :term:`predict`. By default, :term:`predict_proba` is tried first - and we revert to :term:`decision_function` if it doesn't exist. If - ``method`` is 'recursion', the response is always the output of - :term:`decision_function`. - - n_cols : int, optional (default=3) - The maximum number of columns in the grid plot. Only active when `ax` - is a single axis or `None`. - - grid_resolution : int, optional (default=100) - The number of equally spaced points on the axes of the plots, for each - target feature. - - percentiles : tuple of float, optional (default=(0.05, 0.95)) - The lower and upper percentile used to create the extreme values - for the PDP axes. Must be in [0, 1]. - - method : str, optional (default='auto') - The method to use to calculate the partial dependence predictions: - - - 'recursion' is only supported for gradient boosting estimator (namely - :class:`GradientBoostingClassifier`, - :class:`GradientBoostingRegressor`, - :class:`HistGradientBoostingClassifier`, - :class:`HistGradientBoostingRegressor`) - but is more efficient in terms of speed. - With this method, ``X`` is optional and is only used to build the - grid and the partial dependences are computed using the training - data. This method does not account for the ``init`` predictor of - the boosting process, which may lead to incorrect values (see - warning below. With this method, the target response of a - classifier is always the decision function, not the predicted - probabilities. - - - 'brute' is supported for any estimator, but is more - computationally intensive. - - - 'auto': - - 'recursion' is used for estimators that supports it. - - 'brute' is used for all other estimators. - - n_jobs : int, optional (default=None) - The number of CPUs to use to compute the partial dependences. - ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. - ``-1`` means using all processors. See :term:`Glossary ` - for more details. - - verbose : int, optional (default=0) - Verbose output during PD computations. - - fig : Matplotlib figure object, optional (default=None) - A figure object onto which the plots will be drawn, after the figure - has been cleared. By default, a new one is created. - - .. deprecated:: 0.22 - ``fig`` will be removed in 0.24. - - line_kw : dict, optional - Dict with keywords passed to the ``matplotlib.pyplot.plot`` call. - For one-way partial dependence plots. - - contour_kw : dict, optional - Dict with keywords passed to the ``matplotlib.pyplot.contourf`` call. - For two-way partial dependence plots. - - ax : Matplotlib axes or array-like of Matplotlib axes, default=None - - If a single axis is passed in, it is treated as a bounding axes - and a grid of partial dependence plots will be drawn within - these bounds. The `n_cols` parameter controls the number of - columns in the grid. - - If an array-like of axes are passed in, the partial dependence - plots will be drawn directly into these axes. - - If `None`, a figure and a bounding axes is created and treated - as the single axes case. - - .. versionadded:: 0.22 - - Returns - ------- - display: :class:`~sklearn.inspection.PartialDependenceDisplay` - - Examples - -------- - >>> from sklearn.datasets import make_friedman1 - >>> from sklearn.ensemble import GradientBoostingRegressor - >>> X, y = make_friedman1() - >>> clf = GradientBoostingRegressor(n_estimators=10).fit(X, y) - >>> plot_partial_dependence(clf, X, [0, (0, 1)]) #doctest: +SKIP - - See also - -------- - sklearn.inspection.partial_dependence: Return raw partial - dependence values - - Warnings - -------- - The 'recursion' method only works for gradient boosting estimators, and - unlike the 'brute' method, it does not account for the ``init`` - predictor of the boosting process. In practice this will produce the - same values as 'brute' up to a constant offset in the target response, - provided that ``init`` is a consant estimator (which is the default). - However, as soon as ``init`` is not a constant estimator, the partial - dependence values are incorrect for 'recursion'. This is not relevant for - :class:`HistGradientBoostingClassifier - ` and - :class:`HistGradientBoostingRegressor - `, which do not have an - ``init`` parameter. - """ - check_matplotlib_support('plot_partial_dependence') # noqa - import matplotlib.pyplot as plt # noqa - from matplotlib import transforms # noqa - from matplotlib.ticker import MaxNLocator # noqa - from matplotlib.ticker import ScalarFormatter # noqa - - # set target_idx for multi-class estimators - if hasattr(estimator, 'classes_') and np.size(estimator.classes_) > 2: - if target is None: - raise ValueError('target must be specified for multi-class') - target_idx = np.searchsorted(estimator.classes_, target) - if (not (0 <= target_idx < len(estimator.classes_)) or - estimator.classes_[target_idx] != target): - raise ValueError('target not in est.classes_, got {}'.format( - target)) - else: - # regression and binary classification - target_idx = 0 - - # Use check_array only on lists and other non-array-likes / sparse. Do not - # convert DataFrame into a NumPy array. - if not(hasattr(X, '__array__') or sparse.issparse(X)): - X = check_array(X, force_all_finite='allow-nan', dtype=np.object) - n_features = X.shape[1] - - # convert feature_names to list - if feature_names is None: - if hasattr(X, "loc"): - # get the column names for a pandas dataframe - feature_names = X.columns.tolist() - else: - # define a list of numbered indices for a numpy array - feature_names = [str(i) for i in range(n_features)] - elif isinstance(feature_names, np.ndarray): - feature_names = feature_names.tolist() - if len(set(feature_names)) != len(feature_names): - raise ValueError('feature_names should not contain duplicates.') - - def convert_feature(fx): - if isinstance(fx, str): - try: - fx = feature_names.index(fx) - except ValueError: - raise ValueError('Feature %s not in feature_names' % fx) - return int(fx) - - # convert features into a seq of int tuples - tmp_features = [] - for fxs in features: - if isinstance(fxs, (numbers.Integral, str)): - fxs = (fxs,) - try: - fxs = tuple(convert_feature(fx) for fx in fxs) - except TypeError: - raise ValueError('Each entry in features must be either an int, ' - 'a string, or an iterable of size at most 2.') - if not 1 <= np.size(fxs) <= 2: - raise ValueError('Each entry in features must be either an int, ' - 'a string, or an iterable of size at most 2.') - - tmp_features.append(fxs) - - features = tmp_features - - if isinstance(ax, list): - if len(ax) != len(features): - raise ValueError("Expected len(ax) == len(features), " - "got len(ax) = {}".format(len(ax))) - - for i in chain.from_iterable(features): - if i >= len(feature_names): - raise ValueError('All entries of features must be less than ' - 'len(feature_names) = {0}, got {1}.' - .format(len(feature_names), i)) - - # compute averaged predictions - pd_results = Parallel(n_jobs=n_jobs, verbose=verbose)( - delayed(partial_dependence)(estimator, X, fxs, - response_method=response_method, - method=method, - grid_resolution=grid_resolution, - percentiles=percentiles) - for fxs in features) - - # For multioutput regression, we can only check the validity of target - # now that we have the predictions. - # Also note: as multiclass-multioutput classifiers are not supported, - # multiclass and multioutput scenario are mutually exclusive. So there is - # no risk of overwriting target_idx here. - avg_preds, _ = pd_results[0] # checking the first result is enough - if is_regressor(estimator) and avg_preds.shape[0] > 1: - if target is None: - raise ValueError( - 'target must be specified for multi-output regressors') - if not 0 <= target <= avg_preds.shape[0]: - raise ValueError( - 'target must be in [0, n_tasks], got {}.'.format(target)) - target_idx = target - - # get global min and max average predictions of PD grouped by plot type - pdp_lim = {} - for avg_preds, values in pd_results: - min_pd = avg_preds[target_idx].min() - max_pd = avg_preds[target_idx].max() - n_fx = len(values) - old_min_pd, old_max_pd = pdp_lim.get(n_fx, (min_pd, max_pd)) - min_pd = min(min_pd, old_min_pd) - max_pd = max(max_pd, old_max_pd) - pdp_lim[n_fx] = (min_pd, max_pd) - - deciles = {} - for fx in chain.from_iterable(features): - if fx not in deciles: - X_col = _safe_indexing(X, fx, axis=1) - deciles[fx] = mquantiles(X_col, prob=np.arange(0.1, 1.0, 0.1)) - - if fig is not None: - warnings.warn("The fig parameter is deprecated in version " - "0.22 and will be removed in version 0.24", - FutureWarning) - fig.clear() - ax = fig.gca() - - display = PartialDependenceDisplay(pd_results, features, feature_names, - target_idx, pdp_lim, deciles) - return display.plot(ax=ax, n_cols=n_cols, line_kw=line_kw, - contour_kw=contour_kw) - - -class PartialDependenceDisplay: - """Partial Dependence Plot (PDP) visualization. - - It is recommended to use - :func:`~sklearn.inspection.plot_partial_dependence` to create a - :class:`~sklearn.inspection.PartialDependenceDisplay`. All parameters are - stored as attributes. - - Read more in - :ref:`sphx_glr_auto_examples_plot_partial_dependence_visualization_api.py` - and the :ref:`User Guide `. - - .. versionadded:: 0.22 - - Parameters - ---------- - pd_results : list of (ndarray, ndarray) - Results of :func:`~sklearn.inspection.partial_dependence` for - ``features``. Each tuple corresponds to a (averaged_predictions, grid). - - features : list of (int,) or list of (int, int) - Indices of features for a given plot. A tuple of one integer will plot - a partial dependence curve of one feature. A tuple of two integers will - plot a two-way partial dependence curve as a contour plot. - - feature_names : list of str - Feature names corresponding to the indices in ``features``. - - target_idx : int - - - In a multiclass setting, specifies the class for which the PDPs - should be computed. Note that for binary classification, the - positive class (index 1) is always used. - - In a multioutput setting, specifies the task for which the PDPs - should be computed. - - Ignored in binary classification or classical regression settings. - - pdp_lim : dict - Global min and max average predictions, such that all plots will have - the same scale and y limits. `pdp_lim[1]` is the global min and max for - single partial dependence curves. `pdp_lim[2]` is the global min and - max for two-way partial dependence curves. - - deciles : dict - Deciles for feature indices in ``features``. - - Attributes - ---------- - bounding_ax_ : matplotlib Axes or None - If `ax` is an axes or None, the `bounding_ax_` is the axes where the - grid of partial dependence plots are drawn. If `ax` is a list of axes - or a numpy array of axes, `bounding_ax_` is None. - - axes_ : ndarray of matplotlib Axes - If `ax` is an axes or None, `axes_[i, j]` is the axes on the i-th row - and j-th column. If `ax` is a list of axes, `axes_[i]` is the i-th item - in `ax`. Elements that are None corresponds to a nonexisting axes in - that position. - - lines_ : ndarray of matplotlib Artists - If `ax` is an axes or None, `line_[i, j]` is the partial dependence - curve on the i-th row and j-th column. If `ax` is a list of axes, - `lines_[i]` is the partial dependence curve corresponding to the i-th - item in `ax`. Elements that are None corresponds to a nonexisting axes - or an axes that does not include a line plot. - - contours_ : ndarray of matplotlib Artists - If `ax` is an axes or None, `contours_[i, j]` is the partial dependence - plot on the i-th row and j-th column. If `ax` is a list of axes, - `contours_[i]` is the partial dependence plot corresponding to the i-th - item in `ax`. Elements that are None corresponds to a nonexisting axes - or an axes that does not include a contour plot. - - figure_ : matplotlib Figure - Figure containing partial dependence plots. - - """ - def __init__(self, pd_results, features, feature_names, target_idx, - pdp_lim, deciles): - self.pd_results = pd_results - self.features = features - self.feature_names = feature_names - self.target_idx = target_idx - self.pdp_lim = pdp_lim - self.deciles = deciles - - def plot(self, ax=None, n_cols=3, line_kw=None, contour_kw=None): - """Plot partial dependence plots. - - Parameters - ---------- - ax : Matplotlib axes or array-like of Matplotlib axes, default=None - - If a single axis is passed in, it is treated as a bounding axes - and a grid of partial dependence plots will be drawn within - these bounds. The `n_cols` parameter controls the number of - columns in the grid. - - If an array-like of axes are passed in, the partial dependence - plots will be drawn directly into these axes. - - If `None`, a figure and a bounding axes is created and treated - as the single axes case. - - n_cols : int, default=3 - The maximum number of columns in the grid plot. Only active when - `ax` is a single axes or `None`. - - line_kw : dict, default=None - Dict with keywords passed to the `matplotlib.pyplot.plot` call. - For one-way partial dependence plots. - - contour_kw : dict, default=None - Dict with keywords passed to the `matplotlib.pyplot.contourf` - call for two-way partial dependence plots. - - Returns - ------- - display: :class:`~sklearn.inspection.PartialDependenceDisplay` - """ - - check_matplotlib_support("plot_partial_dependence") - import matplotlib.pyplot as plt # noqa - from matplotlib import transforms # noqa - from matplotlib.ticker import MaxNLocator # noqa - from matplotlib.ticker import ScalarFormatter # noqa - from matplotlib.gridspec import GridSpecFromSubplotSpec # noqa - - if line_kw is None: - line_kw = {} - if contour_kw is None: - contour_kw = {} - - if ax is None: - _, ax = plt.subplots() - - default_contour_kws = {"alpha": 0.75} - contour_kw = {**default_contour_kws, **contour_kw} - - n_features = len(self.features) - - if isinstance(ax, plt.Axes): - # If ax has visible==False, it has most likely been set to False - # by a previous call to plot. - if not ax.get_visible(): - raise ValueError("The ax was already used in another plot " - "function, please set ax=display.axes_ " - "instead") - - ax.set_axis_off() - ax.set_visible(False) - self.bounding_ax_ = ax - self.figure_ = ax.figure - - n_cols = min(n_cols, n_features) - n_rows = int(np.ceil(n_features / float(n_cols))) - - self.axes_ = np.empty((n_rows, n_cols), dtype=np.object) - self.lines_ = np.empty((n_rows, n_cols), dtype=np.object) - self.contours_ = np.empty((n_rows, n_cols), dtype=np.object) - - axes_ravel = self.axes_.ravel() - - gs = GridSpecFromSubplotSpec(n_rows, n_cols, - subplot_spec=ax.get_subplotspec()) - for i, spec in zip(range(n_features), gs): - axes_ravel[i] = self.figure_.add_subplot(spec) - - else: # array-like - ax = check_array(ax, dtype=object, ensure_2d=False) - - if ax.ndim == 2: - n_cols = ax.shape[1] - else: - n_cols = None - - if ax.ndim == 1 and ax.shape[0] != n_features: - raise ValueError("Expected len(ax) == len(features), " - "got len(ax) = {}".format(len(ax))) - self.bounding_ax_ = None - self.figure_ = ax.ravel()[0].figure - self.axes_ = ax - self.lines_ = np.empty_like(ax, dtype=np.object) - self.contours_ = np.empty_like(ax, dtype=np.object) - - # create contour levels for two-way plots - if 2 in self.pdp_lim: - Z_level = np.linspace(*self.pdp_lim[2], num=8) - lines_ravel = self.lines_.ravel(order='C') - contours_ravel = self.contours_.ravel(order='C') - - for i, axi, fx, (avg_preds, values) in zip(count(), - self.axes_.ravel(), - self.features, - self.pd_results): - if len(values) == 1: - lines_ravel[i] = axi.plot(values[0], - avg_preds[self.target_idx].ravel(), - **line_kw)[0] - else: - # contour plot - XX, YY = np.meshgrid(values[0], values[1]) - Z = avg_preds[self.target_idx].T - CS = axi.contour(XX, YY, Z, levels=Z_level, linewidths=0.5, - colors='k') - contours_ravel[i] = axi.contourf(XX, YY, Z, levels=Z_level, - vmax=Z_level[-1], - vmin=Z_level[0], - **contour_kw) - axi.clabel(CS, fmt='%2.2f', colors='k', fontsize=10, - inline=True) - - trans = transforms.blended_transform_factory(axi.transData, - axi.transAxes) - ylim = axi.get_ylim() - axi.vlines(self.deciles[fx[0]], 0, 0.05, transform=trans, - color='k') - axi.set_xlabel(self.feature_names[fx[0]]) - axi.set_ylim(ylim) - - if len(values) == 1: - if n_cols is None or i % n_cols == 0: - axi.set_ylabel('Partial dependence') - else: - axi.set_yticklabels([]) - axi.set_ylim(self.pdp_lim[1]) - else: - # contour plot - trans = transforms.blended_transform_factory(axi.transAxes, - axi.transData) - xlim = axi.get_xlim() - axi.hlines(self.deciles[fx[1]], 0, 0.05, transform=trans, - color='k') - # hline erases xlim - axi.set_ylabel(self.feature_names[fx[1]]) - axi.set_xlim(xlim) - return self diff --git a/sklearn/inspection/_permutation_importance.py b/sklearn/inspection/_permutation_importance.py index d71d5fd3f3a68..ff4d9d6738977 100644 --- a/sklearn/inspection/_permutation_importance.py +++ b/sklearn/inspection/_permutation_importance.py @@ -4,41 +4,36 @@ from joblib import delayed from ..metrics import check_scoring +from ..utils import Bunch from ..utils import check_random_state from ..utils import check_array -from ..utils import Bunch - - -def _safe_column_setting(X, col_idx, values): - """Set column on X using `col_idx`""" - if hasattr(X, "iloc"): - X.iloc[:, col_idx] = values - else: - X[:, col_idx] = values - - -def _safe_column_indexing(X, col_idx): - """Return column from X using `col_idx`""" - if hasattr(X, "iloc"): - return X.iloc[:, col_idx].values - else: - return X[:, col_idx] def _calculate_permutation_scores(estimator, X, y, col_idx, random_state, n_repeats, scorer): """Calculate score when `col_idx` is permuted.""" - original_feature = _safe_column_indexing(X, col_idx).copy() - temp = original_feature.copy() + random_state = check_random_state(random_state) + # Work on a copy of X to to ensure thread-safety in case of threading based + # parallelism. Furthermore, making a copy is also useful when the joblib + # backend is 'loky' (default) or the old 'multiprocessing': in those cases, + # if X is large it will be automatically be backed by a readonly memory map + # (memmap). X.copy() on the other hand is always guaranteed to return a + # writable data-structure whose columns can be shuffled inplace. + X_permuted = X.copy() scores = np.zeros(n_repeats) + shuffling_idx = np.arange(X.shape[0]) for n_round in range(n_repeats): - random_state.shuffle(temp) - _safe_column_setting(X, col_idx, temp) - feature_score = scorer(estimator, X, y) + random_state.shuffle(shuffling_idx) + if hasattr(X_permuted, "iloc"): + col = X_permuted.iloc[shuffling_idx, col_idx] + col.index = X_permuted.index + X_permuted.iloc[:, col_idx] = col + else: + X_permuted[:, col_idx] = X_permuted[shuffling_idx, col_idx] + feature_score = scorer(estimator, X_permuted, y) scores[n_round] = feature_score - _safe_column_setting(X, col_idx, original_feature) return scores @@ -83,14 +78,16 @@ def permutation_importance(estimator, X, y, scoring=None, n_repeats=5, `-1` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance, or None, default=None + random_state : int, RandomState instance, default=None Pseudo-random number generator to control the permutations of each - feature. See :term:`random_state`. + feature. + Pass an int to get reproducible results across function calls. + See :term: `Glossary `. Returns ------- - result : Bunch - Dictionary-like object, with attributes: + result : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. importances_mean : ndarray, shape (n_features, ) Mean of feature importance over `n_repeats`. @@ -104,20 +101,22 @@ def permutation_importance(estimator, X, y, scoring=None, n_repeats=5, .. [BRE] L. Breiman, "Random Forests", Machine Learning, 45(1), 5-32, 2001. https://doi.org/10.1023/A:1010933404324 """ - if hasattr(X, "iloc"): - X = X.copy() # Dataframe - else: - X = check_array(X, force_all_finite='allow-nan', dtype=np.object, - copy=True) - + if not hasattr(X, "iloc"): + X = check_array(X, force_all_finite='allow-nan', dtype=None) + + # Precompute random seed from the random state to be used + # to get a fresh independent RandomState instance for each + # parallel call to _calculate_permutation_scores, irrespective of + # the fact that variables are shared or not depending on the active + # joblib backend (sequential, thread-based or process-based). random_state = check_random_state(random_state) - scorer = check_scoring(estimator, scoring=scoring) + random_seed = random_state.randint(np.iinfo(np.int32).max + 1) + scorer = check_scoring(estimator, scoring=scoring) baseline_score = scorer(estimator, X, y) - scores = np.zeros((X.shape[1], n_repeats)) scores = Parallel(n_jobs=n_jobs)(delayed(_calculate_permutation_scores)( - estimator, X, y, col_idx, random_state, n_repeats, scorer + estimator, X, y, col_idx, random_seed, n_repeats, scorer ) for col_idx in range(X.shape[1])) importances = baseline_score - np.array(scores) diff --git a/sklearn/inspection/_plot/__init__.py b/sklearn/inspection/_plot/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/inspection/_plot/partial_dependence.py b/sklearn/inspection/_plot/partial_dependence.py new file mode 100644 index 0000000000000..5dee2750ad37a --- /dev/null +++ b/sklearn/inspection/_plot/partial_dependence.py @@ -0,0 +1,567 @@ +import numbers +from itertools import chain +from itertools import count +import warnings + +import numpy as np +from scipy import sparse +from scipy.stats.mstats import mquantiles +from joblib import Parallel, delayed + +from .. import partial_dependence +from ...base import is_classifier, is_regressor +from ...utils import check_array +from ...utils import check_matplotlib_support # noqa +from ...utils import _safe_indexing + + +def plot_partial_dependence(estimator, X, features, feature_names=None, + target=None, response_method='auto', n_cols=3, + grid_resolution=100, percentiles=(0.05, 0.95), + method='auto', n_jobs=None, verbose=0, fig=None, + line_kw=None, contour_kw=None, ax=None): + """Partial dependence plots. + + The ``len(features)`` plots are arranged in a grid with ``n_cols`` + columns. Two-way partial dependence plots are plotted as contour plots. The + deciles of the feature values will be shown with tick marks on the x-axes + for one-way plots, and on both axes for two-way plots. + + Read more in the :ref:`User Guide `. + + .. note:: + + :func:`plot_partial_dependence` does not support using the same axes + with multiple calls. To plot the the partial dependence for multiple + estimators, please pass the axes created by the first call to the + second call:: + + >>> from sklearn.inspection import plot_partial_dependence + >>> from sklearn.datasets import make_friedman1 + >>> from sklearn.linear_model import LinearRegression + >>> X, y = make_friedman1() + >>> est = LinearRegression().fit(X, y) + >>> disp1 = plot_partial_dependence(est, X) # doctest: +SKIP + >>> disp2 = plot_partial_dependence(est, X, + ... ax=disp1.axes_) # doctest: +SKIP + + .. warning:: + + For :class:`~sklearn.ensemble.GradientBoostingClassifier` and + :class:`~sklearn.ensemble.GradientBoostingRegressor`, the + 'recursion' method (used by default) will not account for the `init` + predictor of the boosting process. In practice, this will produce + the same values as 'brute' up to a constant offset in the target + response, provided that `init` is a constant estimator (which is the + default). However, if `init` is not a constant estimator, the + partial dependence values are incorrect for 'recursion' because the + offset will be sample-dependent. It is preferable to use the 'brute' + method. Note that this only applies to + :class:`~sklearn.ensemble.GradientBoostingClassifier` and + :class:`~sklearn.ensemble.GradientBoostingRegressor`, not to + :class:`~sklearn.ensemble.HistGradientBoostingClassifier` and + :class:`~sklearn.ensemble.HistGradientBoostingRegressor`. + + Parameters + ---------- + estimator : BaseEstimator + A fitted estimator object implementing :term:`predict`, + :term:`predict_proba`, or :term:`decision_function`. + Multioutput-multiclass classifiers are not supported. + + X : {array-like or dataframe} of shape (n_samples, n_features) + ``X`` is used to generate a grid of values for the target + ``features`` (where the partial dependence will be evaluated), and + also to generate values for the complement features when the + `method` is 'brute'. + + features : list of {int, str, pair of int, pair of str} + The target features for which to create the PDPs. + If features[i] is an int or a string, a one-way PDP is created; if + features[i] is a tuple, a two-way PDP is created. Each tuple must be + of size 2. + if any entry is a string, then it must be in ``feature_names``. + + feature_names : array-like of shape (n_features,), dtype=str, default=None + Name of each feature; feature_names[i] holds the name of the feature + with index i. + By default, the name of the feature corresponds to their numerical + index for NumPy array and their column name for pandas dataframe. + + target : int, optional (default=None) + - In a multiclass setting, specifies the class for which the PDPs + should be computed. Note that for binary classification, the + positive class (index 1) is always used. + - In a multioutput setting, specifies the task for which the PDPs + should be computed. + + Ignored in binary classification or classical regression settings. + + response_method : 'auto', 'predict_proba' or 'decision_function', \ + optional (default='auto') + Specifies whether to use :term:`predict_proba` or + :term:`decision_function` as the target response. For regressors + this parameter is ignored and the response is always the output of + :term:`predict`. By default, :term:`predict_proba` is tried first + and we revert to :term:`decision_function` if it doesn't exist. If + ``method`` is 'recursion', the response is always the output of + :term:`decision_function`. + + n_cols : int, optional (default=3) + The maximum number of columns in the grid plot. Only active when `ax` + is a single axis or `None`. + + grid_resolution : int, optional (default=100) + The number of equally spaced points on the axes of the plots, for each + target feature. + + percentiles : tuple of float, optional (default=(0.05, 0.95)) + The lower and upper percentile used to create the extreme values + for the PDP axes. Must be in [0, 1]. + + method : str, optional (default='auto') + The method used to calculate the averaged predictions: + + - 'recursion' is only supported for some tree-based estimators (namely + :class:`~sklearn.ensemble.GradientBoostingClassifier`, + :class:`~sklearn.ensemble.GradientBoostingRegressor`, + :class:`~sklearn.ensemble.HistGradientBoostingClassifier`, + :class:`~sklearn.ensemble.HistGradientBoostingRegressor`, + :class:`~sklearn.tree.DecisionTreeRegressor`, + :class:`~sklearn.ensemble.RandomForestRegressor` + but is more efficient in terms of speed. + With this method, the target response of a + classifier is always the decision function, not the predicted + probabilities. + + - 'brute' is supported for any estimator, but is more + computationally intensive. + + - 'auto': the 'recursion' is used for estimators that support it, + and 'brute' is used otherwise. + + Please see :ref:`this note ` for + differences between the 'brute' and 'recursion' method. + + n_jobs : int, optional (default=None) + The number of CPUs to use to compute the partial dependences. + ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. + ``-1`` means using all processors. See :term:`Glossary ` + for more details. + + verbose : int, optional (default=0) + Verbose output during PD computations. + + fig : Matplotlib figure object, optional (default=None) + A figure object onto which the plots will be drawn, after the figure + has been cleared. By default, a new one is created. + + .. deprecated:: 0.22 + ``fig`` will be removed in 0.24. + + line_kw : dict, optional + Dict with keywords passed to the ``matplotlib.pyplot.plot`` call. + For one-way partial dependence plots. + + contour_kw : dict, optional + Dict with keywords passed to the ``matplotlib.pyplot.contourf`` call. + For two-way partial dependence plots. + + ax : Matplotlib axes or array-like of Matplotlib axes, default=None + - If a single axis is passed in, it is treated as a bounding axes + and a grid of partial dependence plots will be drawn within + these bounds. The `n_cols` parameter controls the number of + columns in the grid. + - If an array-like of axes are passed in, the partial dependence + plots will be drawn directly into these axes. + - If `None`, a figure and a bounding axes is created and treated + as the single axes case. + + .. versionadded:: 0.22 + + Returns + ------- + display: :class:`~sklearn.inspection.PartialDependenceDisplay` + + Examples + -------- + >>> from sklearn.datasets import make_friedman1 + >>> from sklearn.ensemble import GradientBoostingRegressor + >>> X, y = make_friedman1() + >>> clf = GradientBoostingRegressor(n_estimators=10).fit(X, y) + >>> plot_partial_dependence(clf, X, [0, (0, 1)]) #doctest: +SKIP + + See also + -------- + sklearn.inspection.partial_dependence: Return raw partial + dependence values + """ + check_matplotlib_support('plot_partial_dependence') # noqa + import matplotlib.pyplot as plt # noqa + from matplotlib import transforms # noqa + from matplotlib.ticker import MaxNLocator # noqa + from matplotlib.ticker import ScalarFormatter # noqa + + # set target_idx for multi-class estimators + if hasattr(estimator, 'classes_') and np.size(estimator.classes_) > 2: + if target is None: + raise ValueError('target must be specified for multi-class') + target_idx = np.searchsorted(estimator.classes_, target) + if (not (0 <= target_idx < len(estimator.classes_)) or + estimator.classes_[target_idx] != target): + raise ValueError('target not in est.classes_, got {}'.format( + target)) + else: + # regression and binary classification + target_idx = 0 + + # Use check_array only on lists and other non-array-likes / sparse. Do not + # convert DataFrame into a NumPy array. + if not(hasattr(X, '__array__') or sparse.issparse(X)): + X = check_array(X, force_all_finite='allow-nan', dtype=np.object) + n_features = X.shape[1] + + # convert feature_names to list + if feature_names is None: + if hasattr(X, "loc"): + # get the column names for a pandas dataframe + feature_names = X.columns.tolist() + else: + # define a list of numbered indices for a numpy array + feature_names = [str(i) for i in range(n_features)] + elif hasattr(feature_names, "tolist"): + # convert numpy array or pandas index to a list + feature_names = feature_names.tolist() + if len(set(feature_names)) != len(feature_names): + raise ValueError('feature_names should not contain duplicates.') + + def convert_feature(fx): + if isinstance(fx, str): + try: + fx = feature_names.index(fx) + except ValueError: + raise ValueError('Feature %s not in feature_names' % fx) + return int(fx) + + # convert features into a seq of int tuples + tmp_features = [] + for fxs in features: + if isinstance(fxs, (numbers.Integral, str)): + fxs = (fxs,) + try: + fxs = tuple(convert_feature(fx) for fx in fxs) + except TypeError: + raise ValueError('Each entry in features must be either an int, ' + 'a string, or an iterable of size at most 2.') + if not 1 <= np.size(fxs) <= 2: + raise ValueError('Each entry in features must be either an int, ' + 'a string, or an iterable of size at most 2.') + + tmp_features.append(fxs) + + features = tmp_features + + # Early exit if the axes does not have the correct number of axes + if ax is not None and not isinstance(ax, plt.Axes): + axes = np.asarray(ax, dtype=object) + if axes.size != len(features): + raise ValueError("Expected ax to have {} axes, got {}".format( + len(features), axes.size)) + + for i in chain.from_iterable(features): + if i >= len(feature_names): + raise ValueError('All entries of features must be less than ' + 'len(feature_names) = {0}, got {1}.' + .format(len(feature_names), i)) + + # compute averaged predictions + pd_results = Parallel(n_jobs=n_jobs, verbose=verbose)( + delayed(partial_dependence)(estimator, X, fxs, + response_method=response_method, + method=method, + grid_resolution=grid_resolution, + percentiles=percentiles) + for fxs in features) + + # For multioutput regression, we can only check the validity of target + # now that we have the predictions. + # Also note: as multiclass-multioutput classifiers are not supported, + # multiclass and multioutput scenario are mutually exclusive. So there is + # no risk of overwriting target_idx here. + avg_preds, _ = pd_results[0] # checking the first result is enough + if is_regressor(estimator) and avg_preds.shape[0] > 1: + if target is None: + raise ValueError( + 'target must be specified for multi-output regressors') + if not 0 <= target <= avg_preds.shape[0]: + raise ValueError( + 'target must be in [0, n_tasks], got {}.'.format(target)) + target_idx = target + + # get global min and max average predictions of PD grouped by plot type + pdp_lim = {} + for avg_preds, values in pd_results: + min_pd = avg_preds[target_idx].min() + max_pd = avg_preds[target_idx].max() + n_fx = len(values) + old_min_pd, old_max_pd = pdp_lim.get(n_fx, (min_pd, max_pd)) + min_pd = min(min_pd, old_min_pd) + max_pd = max(max_pd, old_max_pd) + pdp_lim[n_fx] = (min_pd, max_pd) + + deciles = {} + for fx in chain.from_iterable(features): + if fx not in deciles: + X_col = _safe_indexing(X, fx, axis=1) + deciles[fx] = mquantiles(X_col, prob=np.arange(0.1, 1.0, 0.1)) + + if fig is not None: + warnings.warn("The fig parameter is deprecated in version " + "0.22 and will be removed in version 0.24", + FutureWarning) + fig.clear() + ax = fig.gca() + + display = PartialDependenceDisplay(pd_results, features, feature_names, + target_idx, pdp_lim, deciles) + return display.plot(ax=ax, n_cols=n_cols, line_kw=line_kw, + contour_kw=contour_kw) + + +class PartialDependenceDisplay: + """Partial Dependence Plot (PDP) visualization. + + It is recommended to use + :func:`~sklearn.inspection.plot_partial_dependence` to create a + :class:`~sklearn.inspection.PartialDependenceDisplay`. All parameters are + stored as attributes. + + Read more in + :ref:`sphx_glr_auto_examples_plot_partial_dependence_visualization_api.py` + and the :ref:`User Guide `. + + .. versionadded:: 0.22 + + Parameters + ---------- + pd_results : list of (ndarray, ndarray) + Results of :func:`~sklearn.inspection.partial_dependence` for + ``features``. Each tuple corresponds to a (averaged_predictions, grid). + + features : list of (int,) or list of (int, int) + Indices of features for a given plot. A tuple of one integer will plot + a partial dependence curve of one feature. A tuple of two integers will + plot a two-way partial dependence curve as a contour plot. + + feature_names : list of str + Feature names corresponding to the indices in ``features``. + + target_idx : int + + - In a multiclass setting, specifies the class for which the PDPs + should be computed. Note that for binary classification, the + positive class (index 1) is always used. + - In a multioutput setting, specifies the task for which the PDPs + should be computed. + + Ignored in binary classification or classical regression settings. + + pdp_lim : dict + Global min and max average predictions, such that all plots will have + the same scale and y limits. `pdp_lim[1]` is the global min and max for + single partial dependence curves. `pdp_lim[2]` is the global min and + max for two-way partial dependence curves. + + deciles : dict + Deciles for feature indices in ``features``. + + Attributes + ---------- + bounding_ax_ : matplotlib Axes or None + If `ax` is an axes or None, the `bounding_ax_` is the axes where the + grid of partial dependence plots are drawn. If `ax` is a list of axes + or a numpy array of axes, `bounding_ax_` is None. + + axes_ : ndarray of matplotlib Axes + If `ax` is an axes or None, `axes_[i, j]` is the axes on the i-th row + and j-th column. If `ax` is a list of axes, `axes_[i]` is the i-th item + in `ax`. Elements that are None corresponds to a nonexisting axes in + that position. + + lines_ : ndarray of matplotlib Artists + If `ax` is an axes or None, `line_[i, j]` is the partial dependence + curve on the i-th row and j-th column. If `ax` is a list of axes, + `lines_[i]` is the partial dependence curve corresponding to the i-th + item in `ax`. Elements that are None corresponds to a nonexisting axes + or an axes that does not include a line plot. + + contours_ : ndarray of matplotlib Artists + If `ax` is an axes or None, `contours_[i, j]` is the partial dependence + plot on the i-th row and j-th column. If `ax` is a list of axes, + `contours_[i]` is the partial dependence plot corresponding to the i-th + item in `ax`. Elements that are None corresponds to a nonexisting axes + or an axes that does not include a contour plot. + + figure_ : matplotlib Figure + Figure containing partial dependence plots. + + """ + def __init__(self, pd_results, features, feature_names, target_idx, + pdp_lim, deciles): + self.pd_results = pd_results + self.features = features + self.feature_names = feature_names + self.target_idx = target_idx + self.pdp_lim = pdp_lim + self.deciles = deciles + + def plot(self, ax=None, n_cols=3, line_kw=None, contour_kw=None): + """Plot partial dependence plots. + + Parameters + ---------- + ax : Matplotlib axes or array-like of Matplotlib axes, default=None + - If a single axis is passed in, it is treated as a bounding axes + and a grid of partial dependence plots will be drawn within + these bounds. The `n_cols` parameter controls the number of + columns in the grid. + - If an array-like of axes are passed in, the partial dependence + plots will be drawn directly into these axes. + - If `None`, a figure and a bounding axes is created and treated + as the single axes case. + + n_cols : int, default=3 + The maximum number of columns in the grid plot. Only active when + `ax` is a single axes or `None`. + + line_kw : dict, default=None + Dict with keywords passed to the `matplotlib.pyplot.plot` call. + For one-way partial dependence plots. + + contour_kw : dict, default=None + Dict with keywords passed to the `matplotlib.pyplot.contourf` + call for two-way partial dependence plots. + + Returns + ------- + display: :class:`~sklearn.inspection.PartialDependenceDisplay` + """ + + check_matplotlib_support("plot_partial_dependence") + import matplotlib.pyplot as plt # noqa + from matplotlib import transforms # noqa + from matplotlib.ticker import MaxNLocator # noqa + from matplotlib.ticker import ScalarFormatter # noqa + from matplotlib.gridspec import GridSpecFromSubplotSpec # noqa + + if line_kw is None: + line_kw = {} + if contour_kw is None: + contour_kw = {} + + if ax is None: + _, ax = plt.subplots() + + default_contour_kws = {"alpha": 0.75} + contour_kw = {**default_contour_kws, **contour_kw} + + n_features = len(self.features) + + if isinstance(ax, plt.Axes): + # If ax was set off, it has most likely been set to off + # by a previous call to plot. + if not ax.axison: + raise ValueError("The ax was already used in another plot " + "function, please set ax=display.axes_ " + "instead") + + ax.set_axis_off() + self.bounding_ax_ = ax + self.figure_ = ax.figure + + n_cols = min(n_cols, n_features) + n_rows = int(np.ceil(n_features / float(n_cols))) + + self.axes_ = np.empty((n_rows, n_cols), dtype=np.object) + self.lines_ = np.empty((n_rows, n_cols), dtype=np.object) + self.contours_ = np.empty((n_rows, n_cols), dtype=np.object) + + axes_ravel = self.axes_.ravel() + + gs = GridSpecFromSubplotSpec(n_rows, n_cols, + subplot_spec=ax.get_subplotspec()) + for i, spec in zip(range(n_features), gs): + axes_ravel[i] = self.figure_.add_subplot(spec) + + else: # array-like + ax = np.asarray(ax, dtype=object) + if ax.size != n_features: + raise ValueError("Expected ax to have {} axes, got {}" + .format(n_features, ax.size)) + + if ax.ndim == 2: + n_cols = ax.shape[1] + else: + n_cols = None + + self.bounding_ax_ = None + self.figure_ = ax.ravel()[0].figure + self.axes_ = ax + self.lines_ = np.empty_like(ax, dtype=np.object) + self.contours_ = np.empty_like(ax, dtype=np.object) + + # create contour levels for two-way plots + if 2 in self.pdp_lim: + Z_level = np.linspace(*self.pdp_lim[2], num=8) + lines_ravel = self.lines_.ravel(order='C') + contours_ravel = self.contours_.ravel(order='C') + + for i, axi, fx, (avg_preds, values) in zip(count(), + self.axes_.ravel(), + self.features, + self.pd_results): + if len(values) == 1: + lines_ravel[i] = axi.plot(values[0], + avg_preds[self.target_idx].ravel(), + **line_kw)[0] + else: + # contour plot + XX, YY = np.meshgrid(values[0], values[1]) + Z = avg_preds[self.target_idx].T + CS = axi.contour(XX, YY, Z, levels=Z_level, linewidths=0.5, + colors='k') + contours_ravel[i] = axi.contourf(XX, YY, Z, levels=Z_level, + vmax=Z_level[-1], + vmin=Z_level[0], + **contour_kw) + axi.clabel(CS, fmt='%2.2f', colors='k', fontsize=10, + inline=True) + + trans = transforms.blended_transform_factory(axi.transData, + axi.transAxes) + ylim = axi.get_ylim() + axi.vlines(self.deciles[fx[0]], 0, 0.05, transform=trans, + color='k') + axi.set_ylim(ylim) + + # Set xlabel if it is not already set + if not axi.get_xlabel(): + axi.set_xlabel(self.feature_names[fx[0]]) + + if len(values) == 1: + if n_cols is None or i % n_cols == 0: + axi.set_ylabel('Partial dependence') + else: + axi.set_yticklabels([]) + axi.set_ylim(self.pdp_lim[1]) + else: + # contour plot + trans = transforms.blended_transform_factory(axi.transAxes, + axi.transData) + xlim = axi.get_xlim() + axi.hlines(self.deciles[fx[1]], 0, 0.05, transform=trans, + color='k') + # hline erases xlim + axi.set_ylabel(self.feature_names[fx[1]]) + axi.set_xlim(xlim) + return self diff --git a/sklearn/inspection/_plot/tests/__init__.py b/sklearn/inspection/_plot/tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/inspection/tests/test_plot_partial_dependence.py b/sklearn/inspection/_plot/tests/test_plot_partial_dependence.py similarity index 82% rename from sklearn/inspection/tests/test_plot_partial_dependence.py rename to sklearn/inspection/_plot/tests/test_plot_partial_dependence.py index 0adf066ec83cd..abae91d4d2642 100644 --- a/sklearn/inspection/tests/test_plot_partial_dependence.py +++ b/sklearn/inspection/_plot/tests/test_plot_partial_dependence.py @@ -10,9 +10,17 @@ from sklearn.ensemble import GradientBoostingRegressor from sklearn.ensemble import GradientBoostingClassifier from sklearn.linear_model import LinearRegression +from sklearn.utils._testing import _convert_container + from sklearn.inspection import plot_partial_dependence +# TODO: Remove when https://github.com/numpy/numpy/issues/14397 is resolved +pytestmark = pytest.mark.filterwarnings( + "ignore:In future, it will be an error for 'np.bool_':DeprecationWarning:" + "matplotlib.*") + + @pytest.fixture(scope="module") def boston(): return load_boston() @@ -86,12 +94,15 @@ def test_plot_partial_dependence(grid_resolution, pyplot, clf_boston, boston): @pytest.mark.parametrize( - "input_type, use_feature_names", - [('dataframe', False), ('dataframe', True), - ('list', True), ('array', True)] + "input_type, feature_names_type", + [('dataframe', None), + ('dataframe', 'list'), ('list', 'list'), ('array', 'list'), + ('dataframe', 'array'), ('list', 'array'), ('array', 'array'), + ('dataframe', 'series'), ('list', 'series'), ('array', 'series'), + ('dataframe', 'index'), ('list', 'index'), ('array', 'index')] ) def test_plot_partial_dependence_str_features(pyplot, clf_boston, boston, - input_type, use_feature_names): + input_type, feature_names_type): if input_type == 'dataframe': pd = pytest.importorskip("pandas") X = pd.DataFrame(boston.data, columns=boston.feature_names) @@ -99,7 +110,12 @@ def test_plot_partial_dependence_str_features(pyplot, clf_boston, boston, X = boston.data.tolist() else: X = boston.data - feature_names = boston.feature_names if use_feature_names else None + + if feature_names_type is None: + feature_names = None + else: + feature_names = _convert_container(boston.feature_names, + feature_names_type) grid_resolution = 25 # check with str features and array feature names and single column @@ -206,30 +222,44 @@ def test_plot_partial_dependence_passing_numpy_axes(pyplot, clf_boston, assert len(disp2.axes_[0, 1].get_lines()) == 2 +@pytest.mark.parametrize("nrows, ncols", [(2, 2), (3, 1)]) def test_plot_partial_dependence_incorrent_num_axes(pyplot, clf_boston, - boston): - grid_resolution = 25 - fig, (ax1, ax2, ax3) = pyplot.subplots(1, 3) + boston, nrows, ncols): + grid_resolution = 5 + fig, axes = pyplot.subplots(nrows, ncols) + axes_formats = [list(axes.ravel()), tuple(axes.ravel()), axes] - msg = r"Expected len\(ax\) == len\(features\), got len\(ax\) = 3" - with pytest.raises(ValueError, match=msg): - plot_partial_dependence(clf_boston, boston.data, - ['CRIM', ('CRIM', 'ZN')], - grid_resolution=grid_resolution, - feature_names=boston.feature_names, - ax=[ax1, ax2, ax3]) + msg = "Expected ax to have 2 axes, got {}".format(nrows * ncols) disp = plot_partial_dependence(clf_boston, boston.data, - ['CRIM', ('CRIM', 'ZN')], + ['CRIM', 'ZN'], grid_resolution=grid_resolution, feature_names=boston.feature_names) - with pytest.raises(ValueError, match=msg): - disp.plot(ax=[ax1, ax2, ax3]) + for ax_format in axes_formats: + with pytest.raises(ValueError, match=msg): + plot_partial_dependence(clf_boston, boston.data, + ['CRIM', 'ZN'], + grid_resolution=grid_resolution, + feature_names=boston.feature_names, + ax=ax_format) + + # with axes object + with pytest.raises(ValueError, match=msg): + disp.plot(ax=ax_format) def test_plot_partial_dependence_with_same_axes(pyplot, clf_boston, boston): - # The first call to `plot_*` will plot the axes + # The first call to plot_partial_dependence will create two new axes to + # place in the space of the passed in axes, which results in a total of + # three axes in the figure. + # Currently the API does not allow for the second call to + # plot_partial_dependence to use the same axes again, because it will + # create two new axes in the space resulting in five axes. To get the + # expected behavior one needs to pass the generated axes into the second + # call: + # disp1 = plot_partial_dependence(...) + # disp2 = plot_partial_dependence(..., ax=disp1.axes_) grid_resolution = 25 fig, ax = pyplot.subplots() @@ -247,6 +277,24 @@ def test_plot_partial_dependence_with_same_axes(pyplot, clf_boston, boston): feature_names=boston.feature_names, ax=ax) +def test_plot_partial_dependence_feature_name_reuse(pyplot, clf_boston, + boston): + # second call to plot does not change the feature names from the first + # call + + feature_names = boston.feature_names + disp = plot_partial_dependence(clf_boston, boston.data, + [0, 1], + grid_resolution=10, + feature_names=feature_names) + + plot_partial_dependence(clf_boston, boston.data, [0, 1], + grid_resolution=10, ax=disp.axes_) + + for i, ax in enumerate(disp.axes_.ravel()): + assert ax.get_xlabel() == feature_names[i] + + def test_plot_partial_dependence_multiclass(pyplot): grid_resolution = 25 clf_int = GradientBoostingClassifier(n_estimators=10, random_state=1) diff --git a/sklearn/inspection/setup.py b/sklearn/inspection/setup.py new file mode 100644 index 0000000000000..e4f629d9ba0f0 --- /dev/null +++ b/sklearn/inspection/setup.py @@ -0,0 +1,17 @@ +from numpy.distutils.misc_util import Configuration + + +def configuration(parent_package="", top_path=None): + config = Configuration("inspection", parent_package, top_path) + + config.add_subpackage('_plot') + config.add_subpackage('_plot.tests') + + config.add_subpackage('tests') + + return config + + +if __name__ == "__main__": + from numpy.distutils.core import setup + setup(**configuration().todict()) diff --git a/sklearn/inspection/tests/test_partial_dependence.py b/sklearn/inspection/tests/test_partial_dependence.py index fd154356b9e0c..530a53b83dce4 100644 --- a/sklearn/inspection/tests/test_partial_dependence.py +++ b/sklearn/inspection/tests/test_partial_dependence.py @@ -14,6 +14,7 @@ ) from sklearn.ensemble import GradientBoostingClassifier from sklearn.ensemble import GradientBoostingRegressor +from sklearn.ensemble import RandomForestRegressor from sklearn.experimental import enable_hist_gradient_boosting # noqa from sklearn.ensemble import HistGradientBoostingClassifier from sklearn.ensemble import HistGradientBoostingRegressor @@ -36,6 +37,9 @@ from sklearn.utils._testing import assert_allclose from sklearn.utils._testing import assert_array_equal from sklearn.utils._testing import ignore_warnings +from sklearn.utils import _IS_32BIT +from sklearn.utils.validation import check_random_state +from sklearn.tree.tests.test_tree import assert_is_subtree # toy sample @@ -174,6 +178,11 @@ def test_partial_dependence_helpers(est, method, target_feature): # samples. # This also checks that the brute and recursion methods give the same # output. + # Note that even on the trainset, the brute and the recursion methods + # aren't always strictly equivalent, in particular when the slow method + # generates unrealistic samples that have low mass in the joint + # distribution of the input features, and when some of the features are + # dependent. Hence the high tolerance on the checks. X, y = make_regression(random_state=0, n_features=5, n_informative=5) # The 'init' estimator for GBDT (here the average prediction) isn't taken @@ -206,6 +215,71 @@ def test_partial_dependence_helpers(est, method, target_feature): assert np.allclose(pdp, mean_predictions, rtol=rtol) +@pytest.mark.parametrize('seed', range(1)) +def test_recursion_decision_tree_vs_forest_and_gbdt(seed): + # Make sure that the recursion method gives the same results on a + # DecisionTreeRegressor and a GradientBoostingRegressor or a + # RandomForestRegressor with 1 tree and equivalent parameters. + + rng = np.random.RandomState(seed) + + # Purely random dataset to avoid correlated features + n_samples = 1000 + n_features = 5 + X = rng.randn(n_samples, n_features) + y = rng.randn(n_samples) * 10 + + # The 'init' estimator for GBDT (here the average prediction) isn't taken + # into account with the recursion method, for technical reasons. We set + # the mean to 0 to that this 'bug' doesn't have any effect. + y = y - y.mean() + + # set max_depth not too high to avoid splits with same gain but different + # features + max_depth = 5 + + tree_seed = 0 + forest = RandomForestRegressor(n_estimators=1, max_features=None, + bootstrap=False, max_depth=max_depth, + random_state=tree_seed) + # The forest will use ensemble.base._set_random_states to set the + # random_state of the tree sub-estimator. We simulate this here to have + # equivalent estimators. + equiv_random_state = check_random_state(tree_seed).randint( + np.iinfo(np.int32).max) + gbdt = GradientBoostingRegressor(n_estimators=1, learning_rate=1, + criterion='mse', max_depth=max_depth, + random_state=equiv_random_state) + tree = DecisionTreeRegressor(max_depth=max_depth, + random_state=equiv_random_state) + + forest.fit(X, y) + gbdt.fit(X, y) + tree.fit(X, y) + + # sanity check: if the trees aren't the same, the PD values won't be equal + try: + assert_is_subtree(tree.tree_, gbdt[0, 0].tree_) + assert_is_subtree(tree.tree_, forest[0].tree_) + except AssertionError: + # For some reason the trees aren't exactly equal on 32bits, so the PDs + # cannot be equal either. See + # https://github.com/scikit-learn/scikit-learn/issues/8853 + assert _IS_32BIT, "this should only fail on 32 bit platforms" + return + + grid = rng.randn(50).reshape(-1, 1) + for f in range(n_features): + features = np.array([f], dtype=np.int32) + + pdp_forest = _partial_dependence_recursion(forest, grid, features) + pdp_gbdt = _partial_dependence_recursion(gbdt, grid, features) + pdp_tree = _partial_dependence_recursion(tree, grid, features) + + np.testing.assert_allclose(pdp_gbdt, pdp_tree) + np.testing.assert_allclose(pdp_forest, pdp_tree) + + @pytest.mark.parametrize('est', ( GradientBoostingClassifier(random_state=0), HistGradientBoostingClassifier(random_state=0), @@ -236,8 +310,9 @@ def test_recursion_decision_function(est, target_feature): LinearRegression(), GradientBoostingRegressor(random_state=0), HistGradientBoostingRegressor(random_state=0, min_samples_leaf=1, - max_leaf_nodes=None, max_iter=1)) -) + max_leaf_nodes=None, max_iter=1), + DecisionTreeRegressor(random_state=0), +)) @pytest.mark.parametrize('power', (1, 2)) def test_partial_dependence_easy_target(est, power): # If the target y only depends on one feature in an obvious way (linear or @@ -445,6 +520,16 @@ def test_partial_dependence_sample_weight(): assert np.corrcoef(pdp, values)[0, 1] > 0.99 +def test_hist_gbdt_sw_not_supported(): + # TODO: remove/fix when PDP supports HGBT with sample weights + clf = HistGradientBoostingRegressor(random_state=1) + clf.fit(X, y, sample_weight=np.ones(len(X))) + + with pytest.raises(NotImplementedError, + match="does not support partial dependence"): + partial_dependence(clf, X, features=[1]) + + # TODO: Remove in 0.24 when DummyClassifier's `strategy` default updates @ignore_warnings(category=FutureWarning) def test_partial_dependence_pipeline(): diff --git a/sklearn/inspection/tests/test_permutation_importance.py b/sklearn/inspection/tests/test_permutation_importance.py index 671a1e11b1fec..c13638b2fc0c7 100644 --- a/sklearn/inspection/tests/test_permutation_importance.py +++ b/sklearn/inspection/tests/test_permutation_importance.py @@ -6,17 +6,25 @@ from sklearn.compose import ColumnTransformer from sklearn.datasets import load_boston from sklearn.datasets import load_iris +from sklearn.datasets import make_classification from sklearn.datasets import make_regression +from sklearn.dummy import DummyClassifier from sklearn.ensemble import RandomForestRegressor from sklearn.ensemble import RandomForestClassifier from sklearn.linear_model import LinearRegression from sklearn.linear_model import LogisticRegression from sklearn.impute import SimpleImputer from sklearn.inspection import permutation_importance +from sklearn.model_selection import train_test_split from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import KBinsDiscretizer from sklearn.preprocessing import OneHotEncoder from sklearn.preprocessing import StandardScaler from sklearn.preprocessing import scale +from sklearn.utils import parallel_backend +from sklearn.utils._testing import _convert_container + + @pytest.mark.parametrize("n_jobs", [1, 2]) def test_permutation_importance_correlated_feature_regression(n_jobs): @@ -76,6 +84,79 @@ def test_permutation_importance_correlated_feature_regression_pandas(n_jobs): assert np.all(result.importances_mean[-1] > result.importances_mean[:-1]) +@pytest.mark.parametrize("n_jobs", [1, 2]) +def test_robustness_to_high_cardinality_noisy_feature(n_jobs, seed=42): + # Permutation variable importance should not be affected by the high + # cardinality bias of traditional feature importances, especially when + # computed on a held-out test set: + rng = np.random.RandomState(seed) + n_repeats = 5 + n_samples = 1000 + n_classes = 5 + n_informative_features = 2 + n_noise_features = 1 + n_features = n_informative_features + n_noise_features + + # Generate a multiclass classification dataset and a set of informative + # binary features that can be used to predict some classes of y exactly + # while leaving some classes unexplained to make the problem harder. + classes = np.arange(n_classes) + y = rng.choice(classes, size=n_samples) + X = np.hstack([(y == c).reshape(-1, 1) + for c in classes[:n_informative_features]]) + X = X.astype(np.float32) + + # Not all target classes are explained by the binary class indicator + # features: + assert n_informative_features < n_classes + + # Add 10 other noisy features with high cardinality (numerical) values + # that can be used to overfit the training data. + X = np.concatenate([X, rng.randn(n_samples, n_noise_features)], axis=1) + assert X.shape == (n_samples, n_features) + + # Split the dataset to be able to evaluate on a held-out test set. The + # Test size should be large enough for importance measurements to be + # stable: + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.5, random_state=rng) + clf = RandomForestClassifier(n_estimators=5, random_state=rng) + clf.fit(X_train, y_train) + + # Variable importances computed by impurity decrease on the tree node + # splits often use the noisy features in splits. This can give misleading + # impression that high cardinality noisy variables are the most important: + tree_importances = clf.feature_importances_ + informative_tree_importances = tree_importances[:n_informative_features] + noisy_tree_importances = tree_importances[n_informative_features:] + assert informative_tree_importances.max() < noisy_tree_importances.min() + + # Let's check that permutation-based feature importances do not have this + # problem. + r = permutation_importance(clf, X_test, y_test, n_repeats=n_repeats, + random_state=rng, n_jobs=n_jobs) + + assert r.importances.shape == (X.shape[1], n_repeats) + + # Split the importances between informative and noisy features + informative_importances = r.importances_mean[:n_informative_features] + noisy_importances = r.importances_mean[n_informative_features:] + + # Because we do not have a binary variable explaining each target classes, + # the RF model will have to use the random variable to make some + # (overfitting) splits (as max_depth is not set). Therefore the noisy + # variables will be non-zero but with small values oscillating around + # zero: + assert max(np.abs(noisy_importances)) > 1e-7 + assert noisy_importances.max() < 0.05 + + # The binary features correlated with y should have a higher importance + # than the high cardinality noisy features. + # The maximum test accuracy is 2 / 5 == 0.4, each informative feature + # contributing approximately a bit more than 0.2 of accuracy. + assert informative_importances.min() > 0.15 + + def test_permutation_importance_mixed_types(): rng = np.random.RandomState(42) n_repeats = 4 @@ -150,3 +231,123 @@ def test_permutation_importance_linear_regresssion(): scoring='neg_mean_squared_error') assert_allclose(expected_importances, results.importances_mean, rtol=1e-1, atol=1e-6) + + +def test_permutation_importance_equivalence_sequential_parallel(): + # regression test to make sure that sequential and parallel calls will + # output the same results. + X, y = make_regression(n_samples=500, n_features=10, random_state=0) + lr = LinearRegression().fit(X, y) + + importance_sequential = permutation_importance( + lr, X, y, n_repeats=5, random_state=0, n_jobs=1 + ) + + # First check that the problem is structured enough and that the model is + # complex enough to not yield trivial, constant importances: + imp_min = importance_sequential['importances'].min() + imp_max = importance_sequential['importances'].max() + assert imp_max - imp_min > 0.3 + + # The actually check that parallelism does not impact the results + # either with shared memory (threading) or without isolated memory + # via process-based parallelism using the default backend + # ('loky' or 'multiprocessing') depending on the joblib version: + + # process-based parallelism (by default): + importance_processes = permutation_importance( + lr, X, y, n_repeats=5, random_state=0, n_jobs=2) + assert_allclose( + importance_processes['importances'], + importance_sequential['importances'] + ) + + # thread-based parallelism: + with parallel_backend("threading"): + importance_threading = permutation_importance( + lr, X, y, n_repeats=5, random_state=0, n_jobs=2 + ) + assert_allclose( + importance_threading['importances'], + importance_sequential['importances'] + ) + + +@pytest.mark.parametrize("n_jobs", [None, 1, 2]) +def test_permutation_importance_equivalence_array_dataframe(n_jobs): + # This test checks that the column shuffling logic has the same behavior + # both a dataframe and a simple numpy array. + pd = pytest.importorskip('pandas') + + # regression test to make sure that sequential and parallel calls will + # output the same results. + X, y = make_regression(n_samples=100, n_features=5, random_state=0) + X_df = pd.DataFrame(X) + + # Add a categorical feature that is statistically linked to y: + binner = KBinsDiscretizer(n_bins=3, encode="ordinal") + cat_column = binner.fit_transform(y.reshape(-1, 1)) + + # Concatenate the extra column to the numpy array: integers will be + # cast to float values + X = np.hstack([X, cat_column]) + assert X.dtype.kind == "f" + + # Insert extra column as a non-numpy-native dtype (while keeping backward + # compat for old pandas versions): + if hasattr(pd, "Categorical"): + cat_column = pd.Categorical(cat_column.ravel()) + else: + cat_column = cat_column.ravel() + new_col_idx = len(X_df.columns) + X_df[new_col_idx] = cat_column + assert X_df[new_col_idx].dtype == cat_column.dtype + + # Stich an aribtrary index to the dataframe: + X_df.index = np.arange(len(X_df)).astype(str) + + rf = RandomForestRegressor(n_estimators=5, max_depth=3, random_state=0) + rf.fit(X, y) + + n_repeats = 3 + importance_array = permutation_importance( + rf, X, y, n_repeats=n_repeats, random_state=0, n_jobs=n_jobs + ) + + # First check that the problem is structured enough and that the model is + # complex enough to not yield trivial, constant importances: + imp_min = importance_array['importances'].min() + imp_max = importance_array['importances'].max() + assert imp_max - imp_min > 0.3 + + # Now check that importances computed on dataframe matche the values + # of those computed on the array with the same data. + importance_dataframe = permutation_importance( + rf, X_df, y, n_repeats=n_repeats, random_state=0, n_jobs=n_jobs + ) + assert_allclose( + importance_array['importances'], + importance_dataframe['importances'] + ) + + +@pytest.mark.parametrize("input_type", ["array", "dataframe"]) +def test_permutation_importance_large_memmaped_data(input_type): + # Smoke, non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/15810 + n_samples, n_features = int(5e4), 4 + X, y = make_classification(n_samples=n_samples, n_features=n_features, + random_state=0) + assert X.nbytes > 1e6 # trigger joblib memmaping + + X = _convert_container(X, input_type) + clf = DummyClassifier(strategy='prior').fit(X, y) + + # Actual smoke test: should not raise any error: + n_repeats = 5 + r = permutation_importance(clf, X, y, n_repeats=n_repeats, n_jobs=2) + + # Auxiliary check: DummyClassifier is feature independent: + # permutating feature should not change the predictions + expected_importances = np.zeros((n_features, n_repeats)) + assert_allclose(expected_importances, r.importances) diff --git a/sklearn/isotonic.py b/sklearn/isotonic.py index 173a747b927c2..896044ae9cc6e 100644 --- a/sklearn/isotonic.py +++ b/sklearn/isotonic.py @@ -6,11 +6,13 @@ import numpy as np from scipy import interpolate from scipy.stats import spearmanr +import warnings +import math + from .base import BaseEstimator, TransformerMixin, RegressorMixin from .utils import check_array, check_consistent_length +from .utils.validation import _check_sample_weight, _deprecate_positional_args from ._isotonic import _inplace_contiguous_isotonic_regression, _make_unique -import warnings -import math __all__ = ['check_increasing', 'isotonic_regression', @@ -76,33 +78,26 @@ def check_increasing(x, y): def isotonic_regression(y, sample_weight=None, y_min=None, y_max=None, increasing=True): - """Solve the isotonic regression model:: - - min sum w[i] (y[i] - y_[i]) ** 2 - - subject to y_min = y_[1] <= y_[2] ... <= y_[n] = y_max - - where: - - y[i] are inputs (real numbers) - - y_[i] are fitted - - w[i] are optional strictly positive weights (default to 1.0) + """Solve the isotonic regression model. Read more in the :ref:`User Guide `. Parameters ---------- - y : iterable of floats + y : array-like of shape (n_samples,) The data. - sample_weight : iterable of floats, optional, default: None + sample_weight : array-like of shape (n_samples,), default=None Weights on each point of the regression. If None, weight is set to 1 (equal weights). - y_min : optional, default: None - If not None, set the lowest value of the fit to y_min. + y_min : float, default=None + Lower bound on the lowest predicted value (the minimum value may + still be higher). If not set, defaults to -inf. - y_max : optional, default: None - If not None, set the highest value of the fit to y_max. + y_max : float, default=None + Upper bound on the highest predicted value (the maximum may still be + lower). If not set, defaults to +inf. increasing : boolean, optional, default: True Whether to compute ``y_`` is increasing (if set to True) or decreasing @@ -121,10 +116,8 @@ def isotonic_regression(y, sample_weight=None, y_min=None, y_max=None, order = np.s_[:] if increasing else np.s_[::-1] y = check_array(y, ensure_2d=False, dtype=[np.float64, np.float32]) y = np.array(y[order], dtype=y.dtype) - if sample_weight is None: - sample_weight = np.ones(len(y), dtype=y.dtype) - else: - sample_weight = np.array(sample_weight[order], dtype=y.dtype) + sample_weight = _check_sample_weight(sample_weight, y, dtype=y.dtype) + sample_weight = np.ascontiguousarray(sample_weight[order]) _inplace_contiguous_isotonic_regression(y, sample_weight) if y_min is not None or y_max is not None: @@ -140,46 +133,31 @@ def isotonic_regression(y, sample_weight=None, y_min=None, y_max=None, class IsotonicRegression(RegressorMixin, TransformerMixin, BaseEstimator): """Isotonic regression model. - The isotonic regression optimization problem is defined by:: - - min sum w_i (y[i] - y_[i]) ** 2 - - subject to y_[i] <= y_[j] whenever X[i] <= X[j] - and min(y_) = y_min, max(y_) = y_max - - where: - - ``y[i]`` are inputs (real numbers) - - ``y_[i]`` are fitted - - ``X`` specifies the order. - If ``X`` is non-decreasing then ``y_`` is non-decreasing. - - ``w[i]`` are optional strictly positive weights (default to 1.0) - Read more in the :ref:`User Guide `. .. versionadded:: 0.13 Parameters ---------- - y_min : optional, default: None - If not None, set the lowest value of the fit to y_min. - - y_max : optional, default: None - If not None, set the highest value of the fit to y_max. - - increasing : boolean or string, optional, default: True - If boolean, whether or not to fit the isotonic regression with y - increasing or decreasing. - - The string value "auto" determines whether y should - increase or decrease based on the Spearman correlation estimate's - sign. - - out_of_bounds : string, optional, default: "nan" - The ``out_of_bounds`` parameter handles how x-values outside of the - training domain are handled. When set to "nan", predicted y-values - will be NaN. When set to "clip", predicted y-values will be + y_min : float, default=None + Lower bound on the lowest predicted value (the minimum value may + still be higher). If not set, defaults to -inf. + + y_max : float, default=None + Upper bound on the highest predicted value (the maximum may still be + lower). If not set, defaults to +inf. + + increasing : bool or 'auto', default=True + Determines whether the predictions should be constrained to increase + or decrease with `X`. 'auto' will decide based on the Spearman + correlation estimate's sign. + + out_of_bounds : str, default="nan" + The ``out_of_bounds`` parameter handles how `X` values outside of the + training domain are handled. When set to "nan", predictions + will be NaN. When set to "clip", predictions will be set to the value corresponding to the nearest train interval endpoint. - When set to "raise", allow ``interp1d`` to throw ValueError. + When set to "raise" a `ValueError` is raised. Attributes @@ -193,6 +171,9 @@ class IsotonicRegression(RegressorMixin, TransformerMixin, BaseEstimator): f_ : function The stepwise interpolating function that covers the input domain ``X``. + increasing_ : bool + Inferred value for ``increasing``. + Notes ----- Ties are broken using the secondary method from Leeuw, 1977. @@ -221,7 +202,8 @@ class IsotonicRegression(RegressorMixin, TransformerMixin, BaseEstimator): >>> iso_reg.predict([.1, .2]) array([1.8628..., 3.7256...]) """ - def __init__(self, y_min=None, y_max=None, increasing=True, + @_deprecate_positional_args + def __init__(self, *, y_min=None, y_max=None, increasing=True, out_of_bounds='nan'): self.y_min = y_min self.y_max = y_max @@ -261,13 +243,9 @@ def _build_y(self, X, y, sample_weight, trim_duplicates=True): # If sample_weights is passed, removed zero-weight values and clean # order - if sample_weight is not None: - sample_weight = check_array(sample_weight, ensure_2d=False, - dtype=X.dtype) - mask = sample_weight > 0 - X, y, sample_weight = X[mask], y[mask], sample_weight[mask] - else: - sample_weight = np.ones(len(y), dtype=X.dtype) + sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype) + mask = sample_weight > 0 + X, y, sample_weight = X[mask], y[mask], sample_weight[mask] order = np.lexsort((y, X)) X, y, sample_weight = [array[order] for array in [X, y, sample_weight]] @@ -355,7 +333,7 @@ def transform(self, T): Returns ------- - T_ : array, shape=(n_samples,) + y_pred : ndarray of shape (n_samples,) The transformed data """ @@ -395,7 +373,7 @@ def predict(self, T): Returns ------- - T_ : array, shape=(n_samples,) + y_pred : ndarray of shape (n_samples,) Transformed data. """ return self.transform(T) diff --git a/sklearn/kernel_approximation.py b/sklearn/kernel_approximation.py index 9847ac1940e47..6ae62ce245a56 100644 --- a/sklearn/kernel_approximation.py +++ b/sklearn/kernel_approximation.py @@ -19,6 +19,7 @@ from .utils.extmath import safe_sparse_dot from .utils.validation import check_is_fitted from .metrics.pairwise import pairwise_kernels, KERNEL_PARAMS +from .utils.validation import check_non_negative, _deprecate_positional_args class RBFSampler(TransformerMixin, BaseEstimator): @@ -44,6 +45,18 @@ class RBFSampler(TransformerMixin, BaseEstimator): If None, the random number generator is the RandomState instance used by `np.random`. + Attributes + ---------- + random_offset_ : ndarray of shape (n_components,), dtype=float64 + Random offset used to compute the projection in the `n_components` + dimensions of the feature space. + + random_weights_ : ndarray of shape (n_features, n_components),\ + dtype=float64 + Random projection directions drawn from the Fourier transform + of the RBF kernel. + + Examples -------- >>> from sklearn.kernel_approximation import RBFSampler @@ -68,8 +81,8 @@ class RBFSampler(TransformerMixin, BaseEstimator): Benjamin Recht. (https://people.eecs.berkeley.edu/~brecht/papers/08.rah.rec.nips.pdf) """ - - def __init__(self, gamma=1., n_components=100, random_state=None): + @_deprecate_positional_args + def __init__(self, *, gamma=1., n_components=100, random_state=None): self.gamma = gamma self.n_components = n_components self.random_state = random_state @@ -91,7 +104,7 @@ def fit(self, X, y=None): Returns the transformer. """ - X = check_array(X, accept_sparse='csr') + X = self._validate_data(X, accept_sparse='csr') random_state = check_random_state(self.random_state) n_features = X.shape[1] @@ -174,8 +187,8 @@ class SkewedChi2Sampler(TransformerMixin, BaseEstimator): sklearn.metrics.pairwise.chi2_kernel : The exact chi squared kernel. """ - - def __init__(self, skewedness=1., n_components=100, random_state=None): + @_deprecate_positional_args + def __init__(self, *, skewedness=1., n_components=100, random_state=None): self.skewedness = skewedness self.n_components = n_components self.random_state = random_state @@ -197,7 +210,7 @@ def fit(self, X, y=None): Returns the transformer. """ - X = check_array(X) + X = self._validate_data(X) random_state = check_random_state(self.random_state) n_features = X.shape[1] uniform = random_state.uniform(size=(n_features, self.n_components)) @@ -305,8 +318,8 @@ class AdditiveChi2Sampler(TransformerMixin, BaseEstimator): A. Vedaldi and A. Zisserman, Pattern Analysis and Machine Intelligence, 2011 """ - - def __init__(self, sample_steps=2, sample_interval=None): + @_deprecate_positional_args + def __init__(self, *, sample_steps=2, sample_interval=None): self.sample_steps = sample_steps self.sample_interval = sample_interval @@ -324,7 +337,9 @@ def fit(self, X, y=None): self : object Returns the transformer. """ - check_array(X, accept_sparse='csr') + X = self._validate_data(X, accept_sparse='csr') + check_non_negative(X, 'X in AdditiveChi2Sampler.fit') + if self.sample_interval is None: # See reference, figure 2 c) if self.sample_steps == 1: @@ -359,11 +374,9 @@ def transform(self, X): check_is_fitted(self, msg=msg) X = check_array(X, accept_sparse='csr') + check_non_negative(X, 'X in AdditiveChi2Sampler.transform') sparse = sp.issparse(X) - # check if X has negative values. Doesn't play well with np.log. - if ((X.data if sparse else X) < 0).any(): - raise ValueError("Entries of X must be non-negative.") # zeroth component # 1/cosh = sech # cosh(0) = 1.0 @@ -426,7 +439,8 @@ def _transform_sparse(self, X): return sp.hstack(X_new) def _more_tags(self): - return {'stateless': True} + return {'stateless': True, + 'requires_positive_X': True} class Nystroem(TransformerMixin, BaseEstimator): @@ -520,8 +534,8 @@ class Nystroem(TransformerMixin, BaseEstimator): sklearn.metrics.pairwise.kernel_metrics : List of built-in kernels. """ - - def __init__(self, kernel="rbf", gamma=None, coef0=None, degree=None, + @_deprecate_positional_args + def __init__(self, kernel="rbf", *, gamma=None, coef0=None, degree=None, kernel_params=None, n_components=100, random_state=None): self.kernel = kernel self.gamma = gamma @@ -542,7 +556,7 @@ def fit(self, X, y=None): X : array-like of shape (n_samples, n_features) Training data. """ - X = check_array(X, accept_sparse='csr') + X = self._validate_data(X, accept_sparse='csr') rnd = check_random_state(self.random_state) n_samples = X.shape[0] diff --git a/sklearn/kernel_ridge.py b/sklearn/kernel_ridge.py index d2ae51f466f0b..21c43979c3b1e 100644 --- a/sklearn/kernel_ridge.py +++ b/sklearn/kernel_ridge.py @@ -9,8 +9,8 @@ from .base import BaseEstimator, RegressorMixin, MultiOutputMixin from .metrics.pairwise import pairwise_kernels from .linear_model._ridge import _solve_cholesky_kernel -from .utils import check_array, check_X_y -from .utils.validation import check_is_fitted +from .utils import check_X_y +from .utils.validation import check_is_fitted, _check_sample_weight class KernelRidge(MultiOutputMixin, RegressorMixin, BaseEstimator): @@ -38,19 +38,29 @@ class KernelRidge(MultiOutputMixin, RegressorMixin, BaseEstimator): Parameters ---------- - alpha : {float, array-like}, shape = [n_targets] - Small positive values of alpha improve the conditioning of the problem - and reduce the variance of the estimates. Alpha corresponds to - ``(2*C)^-1`` in other linear models such as LogisticRegression or - LinearSVC. If an array is passed, penalties are assumed to be specific - to the targets. Hence they must correspond in number. + alpha : float or array-like of shape (n_targets,) + Regularization strength; must be a positive float. Regularization + improves the conditioning of the problem and reduces the variance of + the estimates. Larger values specify stronger regularization. + Alpha corresponds to ``1 / (2C)`` in other linear models such as + :class:`~sklearn.linear_model.LogisticRegression` or + :class:`sklearn.svm.LinearSVC`. If an array is passed, penalties are + assumed to be specific to the targets. Hence they must correspond in + number. See :ref:`ridge_regression` for formula. kernel : string or callable, default="linear" - Kernel mapping used internally. A callable should accept two arguments - and the keyword arguments passed to this object as kernel_params, and - should return a floating point number. Set to "precomputed" in - order to pass a precomputed kernel matrix to the estimator - methods instead of samples. + Kernel mapping used internally. This parameter is directly passed to + :class:`sklearn.metrics.pairwise.pairwise_kernel`. + If `kernel` is a string, it must be one of the metrics + in `pairwise.PAIRWISE_KERNEL_FUNCTIONS`. + If `kernel` is "precomputed", X is assumed to be a kernel matrix. + Alternatively, if `kernel` is a callable function, it is called on + each pair of instances (rows) and the resulting value recorded. The + callable should take two rows from X as input and return the + corresponding kernel value as a single number. This means that + callables from :mod:`sklearn.metrics.pairwise` are not allowed, as + they operate on matrices, not single samples. Use the string + identifying the kernel instead. gamma : float, default=None Gamma parameter for the RBF, laplacian, polynomial, exponential chi2 @@ -71,13 +81,13 @@ class KernelRidge(MultiOutputMixin, RegressorMixin, BaseEstimator): Attributes ---------- - dual_coef_ : array, shape = [n_samples] or [n_samples, n_targets] + dual_coef_ : ndarray of shape (n_samples,) or (n_samples, n_targets) Representation of weight vector(s) in kernel space - X_fit_ : {array-like, sparse matrix} of shape (n_samples, n_features) + X_fit_ : {ndarray, sparse matrix} of shape (n_samples, n_features) Training data, which is also required for prediction. If kernel == "precomputed" this is instead the precomputed - training matrix, shape = [n_samples, n_samples]. + training matrix, of shape (n_samples, n_samples). References ---------- @@ -134,8 +144,7 @@ def fit(self, X, y=None, sample_weight=None): ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data. If kernel == "precomputed" this is instead - a precomputed kernel matrix, shape = [n_samples, - n_samples]. + a precomputed kernel matrix, of shape (n_samples, n_samples). y : array-like of shape (n_samples,) or (n_samples, n_targets) Target values @@ -148,10 +157,10 @@ def fit(self, X, y=None, sample_weight=None): self : returns an instance of self. """ # Convert data - X, y = check_X_y(X, y, accept_sparse=("csr", "csc"), multi_output=True, - y_numeric=True) + X, y = self._validate_data(X, y, accept_sparse=("csr", "csc"), + multi_output=True, y_numeric=True) if sample_weight is not None and not isinstance(sample_weight, float): - sample_weight = check_array(sample_weight, ensure_2d=False) + sample_weight = _check_sample_weight(sample_weight, X) K = self._get_kernel(X) alpha = np.atleast_1d(self.alpha) diff --git a/sklearn/linear_model/__init__.py b/sklearn/linear_model/__init__.py index 01c686a69e970..110e0008bccc9 100644 --- a/sklearn/linear_model/__init__.py +++ b/sklearn/linear_model/__init__.py @@ -7,7 +7,6 @@ # complete documentation. from ._base import LinearRegression - from ._bayes import BayesianRidge, ARDRegression from ._least_angle import (Lars, LassoLars, lars_path, lars_path_gram, LarsCV, LassoLarsCV, LassoLarsIC) @@ -15,13 +14,14 @@ lasso_path, enet_path, MultiTaskLasso, MultiTaskElasticNet, MultiTaskElasticNetCV, MultiTaskLassoCV) +from ._glm import (PoissonRegressor, + GammaRegressor, TweedieRegressor) from ._huber import HuberRegressor from ._sgd_fast import Hinge, Log, ModifiedHuber, SquaredLoss, Huber from ._stochastic_gradient import SGDClassifier, SGDRegressor from ._ridge import (Ridge, RidgeCV, RidgeClassifier, RidgeClassifierCV, ridge_regression) -from ._logistic import (LogisticRegression, LogisticRegressionCV, - logistic_regression_path) +from ._logistic import LogisticRegression, LogisticRegressionCV from ._omp import (orthogonal_mp, orthogonal_mp_gram, OrthogonalMatchingPursuit, OrthogonalMatchingPursuitCV) from ._passive_aggressive import PassiveAggressiveClassifier @@ -71,8 +71,10 @@ 'lars_path', 'lars_path_gram', 'lasso_path', - 'logistic_regression_path', 'orthogonal_mp', 'orthogonal_mp_gram', 'ridge_regression', - 'RANSACRegressor'] + 'RANSACRegressor', + 'PoissonRegressor', + 'GammaRegressor', + 'TweedieRegressor'] diff --git a/sklearn/linear_model/_base.py b/sklearn/linear_model/_base.py index 457752d9a560d..d280f9d0f0d81 100644 --- a/sklearn/linear_model/_base.py +++ b/sklearn/linear_model/_base.py @@ -1,5 +1,5 @@ """ -Generalized Linear models. +Generalized Linear Models. """ # Author: Alexandre Gramfort @@ -34,7 +34,7 @@ from ..utils.fixes import sparse_lsqr from ..utils._seq_dataset import ArrayDataset32, CSRDataset32 from ..utils._seq_dataset import ArrayDataset64, CSRDataset64 -from ..utils.validation import check_is_fitted +from ..utils.validation import check_is_fitted, _check_sample_weight from ..preprocessing import normalize as f_normalize # TODO: bayesian_ridge_regression and bayesian_regression_ard @@ -100,7 +100,8 @@ def make_dataset(X, y, sample_weight, random_state=None): def _preprocess_data(X, y, fit_intercept, normalize=False, copy=True, sample_weight=None, return_mean=False, check_input=True): - """ + """Center and scale data. + Centers data to have mean zero along axis 0. If fit_intercept=False or if the X is a sparse matrix, no centering is done, but normalization can still be applied. The function returns the statistics necessary to reconstruct @@ -117,7 +118,6 @@ def _preprocess_data(X, y, fit_intercept, normalize=False, copy=True, This is here because nearly all linear models will want their data to be centered. This function also systematically makes y consistent with X.dtype """ - if isinstance(sample_weight, numbers.Number): sample_weight = None if sample_weight is not None: @@ -181,9 +181,18 @@ def _preprocess_data(X, y, fit_intercept, normalize=False, copy=True, # sample_weight makes the refactoring tricky. def _rescale_data(X, y, sample_weight): - """Rescale data so as to support sample_weight""" + """Rescale data sample-wise by square root of sample_weight. + + For many linear models, this enables easy support for sample_weight. + + Returns + ------- + X_rescaled : {array-like, sparse matrix} + + y_rescaled : {array-like, sparse matrix} + """ n_samples = X.shape[0] - sample_weight = np.array(sample_weight) + sample_weight = np.asarray(sample_weight) if sample_weight.ndim == 0: sample_weight = np.full(n_samples, sample_weight, dtype=sample_weight.dtype) @@ -383,12 +392,12 @@ class LinearRegression(MultiOutputMixin, RegressorMixin, LinearModel): Parameters ---------- - fit_intercept : bool, optional, default True + fit_intercept : bool, default=True Whether to calculate the intercept for this model. If set to False, no intercept will be used in calculations (i.e. data is expected to be centered). - normalize : bool, optional, default False + normalize : bool, default=False This parameter is ignored when ``fit_intercept`` is set to False. If True, the regressors X will be normalized before regression by subtracting the mean and dividing by the l2-norm. @@ -396,10 +405,10 @@ class LinearRegression(MultiOutputMixin, RegressorMixin, LinearModel): :class:`sklearn.preprocessing.StandardScaler` before calling ``fit`` on an estimator with ``normalize=False``. - copy_X : bool, optional, default True + copy_X : bool, default=True If True, X will be copied; else, it may be overwritten. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of jobs to use for the computation. This will only provide speedup for n_targets > 1 and sufficient large problems. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. @@ -408,7 +417,7 @@ class LinearRegression(MultiOutputMixin, RegressorMixin, LinearModel): Attributes ---------- - coef_ : array, shape (n_features, ) or (n_targets, n_features) + coef_ : array of shape (n_features, ) or (n_targets, n_features) Estimated coefficients for the linear regression problem. If multiple targets are passed during the fit (y 2D), this is a 2D array of shape (n_targets, n_features), while if only @@ -417,10 +426,10 @@ class LinearRegression(MultiOutputMixin, RegressorMixin, LinearModel): rank_ : int Rank of matrix `X`. Only available when `X` is dense. - singular_ : array, shape (min(X, y),) + singular_ : array of shape (min(X, y),) Singular values of `X`. Only available when `X` is dense. - intercept_ : float | array, shape = (n_targets,) + intercept_ : float or array of shape (n_targets,) Independent term in the linear model. Set to 0.0 if `fit_intercept = False`. @@ -471,13 +480,13 @@ def fit(self, X, y, sample_weight=None): Parameters ---------- - X : array-like or sparse matrix, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data - y : array_like, shape (n_samples, n_targets) + y : array-like of shape (n_samples,) or (n_samples, n_targets) Target values. Will be cast to X's dtype if necessary - sample_weight : numpy array of shape [n_samples] + sample_weight : array-like of shape (n_samples,), default=None Individual weights for each sample .. versionadded:: 0.17 @@ -489,11 +498,12 @@ def fit(self, X, y, sample_weight=None): """ n_jobs_ = self.n_jobs - X, y = check_X_y(X, y, accept_sparse=['csr', 'csc', 'coo'], - y_numeric=True, multi_output=True) + X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc', 'coo'], + y_numeric=True, multi_output=True) - if sample_weight is not None and np.asarray(sample_weight).ndim > 1: - raise ValueError("Sample weights must be 1D array or scalar") + if sample_weight is not None: + sample_weight = _check_sample_weight(sample_weight, X, + dtype=X.dtype) X, y, X_offset, y_offset, X_scale = self._preprocess_data( X, y, fit_intercept=self.fit_intercept, normalize=self.normalize, @@ -540,8 +550,15 @@ def rmatvec(b): def _pre_fit(X, y, Xy, precompute, normalize, fit_intercept, copy, - check_input=True): - """Aux function used at beginning of fit in linear models""" + check_input=True, sample_weight=None): + """Aux function used at beginning of fit in linear models + + Parameters + ---------- + order : 'F', 'C' or None, default=None + Whether X and y will be forced to be fortran or c-style. Only relevant + if sample_weight is not None. + """ n_samples, n_features = X.shape if sparse.isspmatrix(X): @@ -554,9 +571,11 @@ def _pre_fit(X, y, Xy, precompute, normalize, fit_intercept, copy, # copy was done in fit if necessary X, y, X_offset, y_offset, X_scale = _preprocess_data( X, y, fit_intercept=fit_intercept, normalize=normalize, copy=copy, - check_input=check_input) + check_input=check_input, sample_weight=sample_weight) + if sample_weight is not None: + X, y = _rescale_data(X, y, sample_weight=sample_weight) if hasattr(precompute, '__array__') and ( - fit_intercept and not np.allclose(X_offset, np.zeros(n_features)) or + fit_intercept and not np.allclose(X_offset, np.zeros(n_features)) or normalize and not np.allclose(X_scale, np.ones(n_features))): warnings.warn("Gram matrix was provided but X was centered" " to fit intercept, " diff --git a/sklearn/linear_model/_bayes.py b/sklearn/linear_model/_bayes.py index c97f52ac6778e..c67fc54f43157 100644 --- a/sklearn/linear_model/_bayes.py +++ b/sklearn/linear_model/_bayes.py @@ -14,6 +14,7 @@ from ..utils.extmath import fast_logdet from ..utils import check_X_y from ..utils.fixes import pinvh +from ..utils.validation import _check_sample_weight ############################################################################### @@ -169,7 +170,7 @@ def fit(self, X, y, sample_weight=None): Parameters ---------- - X : ndarray of shape (n_samples,n_features) + X : ndarray of shape (n_samples, n_features) Training data y : ndarray of shape (n_samples,) Target values. Will be cast to X's dtype if necessary @@ -189,7 +190,12 @@ def fit(self, X, y, sample_weight=None): raise ValueError('n_iter should be greater than or equal to 1.' ' Got {!r}.'.format(self.n_iter)) - X, y = check_X_y(X, y, dtype=np.float64, y_numeric=True) + X, y = self._validate_data(X, y, dtype=np.float64, y_numeric=True) + + if sample_weight is not None: + sample_weight = _check_sample_weight(sample_weight, X, + dtype=X.dtype) + X, y, X_offset_, y_offset_, X_scale_ = self._preprocess_data( X, y, self.fit_intercept, self.normalize, self.copy_X, sample_weight=sample_weight) @@ -520,8 +526,8 @@ def fit(self, X, y): ------- self : returns an instance of self. """ - X, y = check_X_y(X, y, dtype=np.float64, y_numeric=True, - ensure_min_samples=2) + X, y = self._validate_data(X, y, dtype=np.float64, y_numeric=True, + ensure_min_samples=2) n_samples, n_features = X.shape coef_ = np.zeros(n_features) diff --git a/sklearn/linear_model/_coordinate_descent.py b/sklearn/linear_model/_coordinate_descent.py index efe5612845157..9281d03710455 100644 --- a/sklearn/linear_model/_coordinate_descent.py +++ b/sklearn/linear_model/_coordinate_descent.py @@ -7,6 +7,7 @@ import sys import warnings +import numbers from abc import ABCMeta, abstractmethod import numpy as np @@ -20,13 +21,57 @@ from ..utils.validation import check_random_state from ..model_selection import check_cv from ..utils.extmath import safe_sparse_dot -from ..utils.fixes import _joblib_parallel_args -from ..utils.validation import check_is_fitted +from ..utils.fixes import _astype_copy_false, _joblib_parallel_args +from ..utils.validation import check_is_fitted, _check_sample_weight from ..utils.validation import column_or_1d from . import _cd_fast as cd_fast +def _set_order(X, y, order='C'): + """Change the order of X and y if necessary. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training data. + + y : ndarray of shape (n_samples,) + Target values. + + order : {None, 'C', 'F'} + If 'C', dense arrays are returned as C-ordered, sparse matrices in csr + format. If 'F', dense arrays are return as F-ordered, sparse matrices + in csc format. + + Returns + ------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training data with guaranteed order. + + y : ndarray of shape (n_samples,) + Target values with guaranteed order. + """ + if order not in [None, 'C', 'F']: + raise ValueError("Unknown value for order. Got {} instead of " + "None, 'C' or 'F'.".format(order)) + sparse_X = sparse.issparse(X) + sparse_y = sparse.issparse(y) + if order is not None: + sparse_format = "csc" if order == "F" else "csr" + if sparse_X: + # As of scipy 1.1.0, new argument copy=False by default. + # This is what we want. + X = X.asformat(sparse_format, **_astype_copy_false(X)) + else: + X = np.asarray(X, order=order) + if sparse_y: + y = y.asformat(sparse_format) + else: + y = np.asarray(y, order=order) + return X, y + + ############################################################################### # Paths functions @@ -36,33 +81,33 @@ def _alpha_grid(X, y, Xy=None, l1_ratio=1.0, fit_intercept=True, Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data. Pass directly as Fortran-contiguous data to avoid unnecessary memory duplication - y : ndarray, shape (n_samples,) + y : ndarray of shape (n_samples,) Target values - Xy : array-like, optional + Xy : array-like of shape (n_features,), default=None Xy = np.dot(X.T, y) that can be precomputed. - l1_ratio : float + l1_ratio : float, default=1.0 The elastic net mixing parameter, with ``0 < l1_ratio <= 1``. For ``l1_ratio = 0`` the penalty is an L2 penalty. (currently not supported) ``For l1_ratio = 1`` it is an L1 penalty. For ``0 < l1_ratio <1``, the penalty is a combination of L1 and L2. - eps : float, optional + eps : float, default=1e-3 Length of the path. ``eps=1e-3`` means that ``alpha_min / alpha_max = 1e-3`` - n_alphas : int, optional + n_alphas : int, default=100 Number of alphas along the regularization path - fit_intercept : boolean, default True + fit_intercept : bool, default=True Whether to fit an intercept or not - normalize : boolean, optional, default False + normalize : bool, default=False This parameter is ignored when ``fit_intercept`` is set to False. If True, the regressors X will be normalized before regression by subtracting the mean and dividing by the l2-norm. @@ -70,7 +115,7 @@ def _alpha_grid(X, y, Xy=None, l1_ratio=1.0, fit_intercept=True, :class:`sklearn.preprocessing.StandardScaler` before calling ``fit`` on an estimator with ``normalize=False``. - copy_X : boolean, optional, default True + copy_X : bool, default=True If ``True``, X will be copied; else, it may be overwritten. """ if l1_ratio == 0: @@ -146,47 +191,50 @@ def lasso_path(X, y, eps=1e-3, n_alphas=100, alphas=None, Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data. Pass directly as Fortran-contiguous data to avoid unnecessary memory duplication. If ``y`` is mono-output then ``X`` can be sparse. - y : ndarray, shape (n_samples,), or (n_samples, n_outputs) + y : {array-like, sparse matrix} of shape (n_samples,) or \ + (n_samples, n_outputs) Target values - eps : float, optional + eps : float, default=1e-3 Length of the path. ``eps=1e-3`` means that ``alpha_min / alpha_max = 1e-3`` - n_alphas : int, optional + n_alphas : int, default=100 Number of alphas along the regularization path - alphas : ndarray, optional + alphas : ndarray, default=None List of alphas where to compute the models. If ``None`` alphas are set automatically - precompute : True | False | 'auto' | array-like + precompute : 'auto', bool or array-like of shape (n_features, n_features),\ + default='auto' Whether to use a precomputed Gram matrix to speed up calculations. If set to ``'auto'`` let us decide. The Gram matrix can also be passed as argument. - Xy : array-like, optional + Xy : array-like of shape (n_features,) or (n_features, n_outputs),\ + default=None Xy = np.dot(X.T, y) that can be precomputed. It is useful only when the Gram matrix is precomputed. - copy_X : boolean, optional, default True + copy_X : bool, default=True If ``True``, X will be copied; else, it may be overwritten. - coef_init : array, shape (n_features, ) | None + coef_init : ndarray of shape (n_features, ), default=None The initial values of the coefficients. - verbose : bool or integer + verbose : bool or int, default=False Amount of verbosity. - return_n_iter : bool + return_n_iter : bool, default=False whether to return the number of iterations or not. - positive : bool, default False + positive : bool, default=False If set to True, forces coefficients to be positive. (Only allowed when ``y.ndim == 1``). @@ -195,17 +243,17 @@ def lasso_path(X, y, eps=1e-3, n_alphas=100, alphas=None, Returns ------- - alphas : array, shape (n_alphas,) + alphas : ndarray of shape (n_alphas,) The alphas along the path where models are computed. - coefs : array, shape (n_features, n_alphas) or \ + coefs : ndarray of shape (n_features, n_alphas) or \ (n_outputs, n_features, n_alphas) Coefficients along the path. - dual_gaps : array, shape (n_alphas,) + dual_gaps : ndarray of shape (n_alphas,) The dual gaps at the end of the optimization for each alpha. - n_iters : array-like, shape (n_alphas,) + n_iters : list of int The number of iterations taken by the coordinate descent optimizer to reach the specified tolerance for each alpha. @@ -294,55 +342,58 @@ def enet_path(X, y, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None, Parameters ---------- - X : {array-like}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data. Pass directly as Fortran-contiguous data to avoid unnecessary memory duplication. If ``y`` is mono-output then ``X`` can be sparse. - y : ndarray, shape (n_samples,) or (n_samples, n_outputs) + y : {array-like, sparse matrix} of shape (n_samples,) or \ + (n_samples, n_outputs) Target values. - l1_ratio : float, optional + l1_ratio : float, default=0.5 Number between 0 and 1 passed to elastic net (scaling between l1 and l2 penalties). ``l1_ratio=1`` corresponds to the Lasso. - eps : float + eps : float, default=1e-3 Length of the path. ``eps=1e-3`` means that ``alpha_min / alpha_max = 1e-3``. - n_alphas : int, optional + n_alphas : int, default=100 Number of alphas along the regularization path. - alphas : ndarray, optional + alphas : ndarray, default=None List of alphas where to compute the models. If None alphas are set automatically. - precompute : True | False | 'auto' | array-like + precompute : 'auto', bool or array-like of shape (n_features, n_features),\ + default='auto' Whether to use a precomputed Gram matrix to speed up calculations. If set to ``'auto'`` let us decide. The Gram matrix can also be passed as argument. - Xy : array-like, optional + Xy : array-like of shape (n_features,) or (n_features, n_outputs),\ + default=None Xy = np.dot(X.T, y) that can be precomputed. It is useful only when the Gram matrix is precomputed. - copy_X : bool, optional, default True + copy_X : bool, default=True If ``True``, X will be copied; else, it may be overwritten. - coef_init : array, shape (n_features, ) | None + coef_init : ndarray of shape (n_features, ), default=None The initial values of the coefficients. - verbose : bool or int + verbose : bool or int, default=False Amount of verbosity. - return_n_iter : bool + return_n_iter : bool, default=False Whether to return the number of iterations or not. - positive : bool, default False + positive : bool, default=False If set to True, forces coefficients to be positive. (Only allowed when ``y.ndim == 1``). - check_input : bool, default True + check_input : bool, default=True Skip input validation checks, including the Gram matrix when provided assuming there are handled by the caller when check_input=False. @@ -351,17 +402,17 @@ def enet_path(X, y, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None, Returns ------- - alphas : array, shape (n_alphas,) + alphas : ndarray of shape (n_alphas,) The alphas along the path where models are computed. - coefs : array, shape (n_features, n_alphas) or \ + coefs : ndarray of shape (n_features, n_alphas) or \ (n_outputs, n_features, n_alphas) Coefficients along the path. - dual_gaps : array, shape (n_alphas,) + dual_gaps : ndarray of shape (n_alphas,) The dual gaps at the end of the optimization for each alpha. - n_iters : array-like, shape (n_alphas,) + n_iters : list of int The number of iterations taken by the coordinate descent optimizer to reach the specified tolerance for each alpha. (Is returned when ``return_n_iter`` is set to True). @@ -526,7 +577,7 @@ class ElasticNet(MultiOutputMixin, RegressorMixin, LinearModel): Parameters ---------- - alpha : float, optional + alpha : float, default=1.0 Constant that multiplies the penalty terms. Defaults to 1.0. See the notes for the exact mathematical meaning of this parameter. ``alpha = 0`` is equivalent to an ordinary least square, @@ -534,17 +585,17 @@ class ElasticNet(MultiOutputMixin, RegressorMixin, LinearModel): reasons, using ``alpha = 0`` with the ``Lasso`` object is not advised. Given this, you should use the :class:`LinearRegression` object. - l1_ratio : float + l1_ratio : float, default=0.5 The ElasticNet mixing parameter, with ``0 <= l1_ratio <= 1``. For ``l1_ratio = 0`` the penalty is an L2 penalty. ``For l1_ratio = 1`` it is an L1 penalty. For ``0 < l1_ratio < 1``, the penalty is a combination of L1 and L2. - fit_intercept : bool + fit_intercept : bool, default=True Whether the intercept should be estimated or not. If ``False``, the data is assumed to be already centered. - normalize : boolean, optional, default False + normalize : bool, default=False This parameter is ignored when ``fit_intercept`` is set to False. If True, the regressors X will be normalized before regression by subtracting the mean and dividing by the l2-norm. @@ -552,40 +603,39 @@ class ElasticNet(MultiOutputMixin, RegressorMixin, LinearModel): :class:`sklearn.preprocessing.StandardScaler` before calling ``fit`` on an estimator with ``normalize=False``. - precompute : True | False | array-like + precompute : bool or array-like of shape (n_features, n_features),\ + default=False Whether to use a precomputed Gram matrix to speed up calculations. The Gram matrix can also be passed as argument. For sparse input this option is always ``True`` to preserve sparsity. - max_iter : int, optional + max_iter : int, default=1000 The maximum number of iterations - copy_X : boolean, optional, default True + copy_X : bool, default=True If ``True``, X will be copied; else, it may be overwritten. - tol : float, optional + tol : float, default=1e-4 The tolerance for the optimization: if the updates are smaller than ``tol``, the optimization code checks the dual gap for optimality and continues until it is smaller than ``tol``. - warm_start : bool, optional + warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit as initialization, otherwise, just erase the previous solution. See :term:`the Glossary `. - positive : bool, optional + positive : bool, default=False When set to ``True``, forces the coefficients to be positive. - random_state : int, RandomState instance or None, optional, default None + random_state : int, RandomState instance, default=None The seed of the pseudo random number generator that selects a random - feature to update. If int, random_state is the seed used by the random - number generator; If RandomState instance, random_state is the random - number generator; If None, the random number generator is the - RandomState instance used by `np.random`. Used when ``selection`` == - 'random'. + feature to update. Used when ``selection`` == 'random'. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. - selection : str, default 'cyclic' + selection : {'cyclic', 'random'}, default='cyclic' If set to 'random', a random coefficient is updated every iteration rather than looping over features sequentially by default. This (setting to 'random') often leads to significantly faster convergence @@ -593,17 +643,17 @@ class ElasticNet(MultiOutputMixin, RegressorMixin, LinearModel): Attributes ---------- - coef_ : array, shape (n_features,) | (n_targets, n_features) + coef_ : ndarray of shape (n_features,) or (n_targets, n_features) parameter vector (w in the cost function formula) - sparse_coef_ : scipy.sparse matrix, shape (n_features, 1) | \ + sparse_coef_ : sparse matrix of shape (n_features, 1) or \ (n_targets, n_features) ``sparse_coef_`` is a readonly property derived from ``coef_`` - intercept_ : float | array, shape (n_targets,) + intercept_ : float or ndarray of shape (n_targets,) independent term in decision function. - n_iter_ : array-like, shape (n_targets,) + n_iter_ : list of int number of iterations run by the coordinate descent solver to reach the specified tolerance. @@ -656,18 +706,22 @@ def __init__(self, alpha=1.0, l1_ratio=0.5, fit_intercept=True, self.random_state = random_state self.selection = selection - def fit(self, X, y, check_input=True): + def fit(self, X, y, sample_weight=None, check_input=True): """Fit model with coordinate descent. Parameters ---------- - X : ndarray or scipy.sparse matrix, (n_samples, n_features) + X : {ndarray, sparse matrix} of (n_samples, n_features) Data - y : ndarray, shape (n_samples,) or (n_samples, n_targets) + y : {ndarray, sparse matrix} of shape (n_samples,) or \ + (n_samples, n_targets) Target. Will be cast to X's dtype if necessary - check_input : boolean, (default=True) + sample_weight : float or array-like of shape (n_samples,), default=None + Sample weight. + + check_input : bool, default=True Allow to bypass several input checking. Don't use this parameter unless you know what you do. @@ -697,24 +751,57 @@ def fit(self, X, y, check_input=True): # when bypassing checks if check_input: X_copied = self.copy_X and self.fit_intercept - X, y = check_X_y(X, y, accept_sparse='csc', - order='F', dtype=[np.float64, np.float32], - copy=X_copied, multi_output=True, y_numeric=True) + X, y = self._validate_data(X, y, accept_sparse='csc', + order='F', + dtype=[np.float64, np.float32], + copy=X_copied, multi_output=True, + y_numeric=True) y = check_array(y, order='F', copy=False, dtype=X.dtype.type, ensure_2d=False) - # Ensure copying happens only once, don't do it again if done above + n_samples, n_features = X.shape + alpha = self.alpha + + if isinstance(sample_weight, numbers.Number): + sample_weight = None + if sample_weight is not None: + if check_input: + if sparse.issparse(X): + raise ValueError("Sample weights do not (yet) support " + "sparse matrices.") + sample_weight = _check_sample_weight(sample_weight, X, + dtype=X.dtype) + # simplify things by rescaling sw to sum up to n_samples + # => np.average(x, weights=sw) = np.mean(sw * x) + sample_weight *= (n_samples / np.sum(sample_weight)) + # Objective function is: + # 1/2 * np.average(squared error, weights=sw) + alpha * penalty + # but coordinate descent minimizes: + # 1/2 * sum(squared error) + alpha * penalty + # enet_path therefore sets alpha = n_samples * alpha + # With sw, enet_path should set alpha = sum(sw) * alpha + # Therefore, we rescale alpha = sum(sw) / n_samples * alpha + # Note: As we rescaled sample_weights to sum up to n_samples, + # we don't need this + # alpha *= np.sum(sample_weight) / n_samples + + # Ensure copying happens only once, don't do it again if done above. + # X and y will be rescaled if sample_weight is not None, order='F' + # ensures that the returned X and y are still F-contiguous. should_copy = self.copy_X and not X_copied X, y, X_offset, y_offset, X_scale, precompute, Xy = \ _pre_fit(X, y, None, self.precompute, self.normalize, self.fit_intercept, copy=should_copy, - check_input=check_input) + check_input=check_input, sample_weight=sample_weight) + # coordinate descent needs F-ordered arrays and _pre_fit might have + # called _rescale_data + if check_input or sample_weight is not None: + X, y = _set_order(X, y, order='F') if y.ndim == 1: y = y[:, np.newaxis] if Xy is not None and Xy.ndim == 1: Xy = Xy[:, np.newaxis] - n_samples, n_features = X.shape n_targets = y.shape[1] if self.selection not in ['cyclic', 'random']: @@ -739,7 +826,7 @@ def fit(self, X, y, check_input=True): _, this_coef, this_dual_gap, this_iter = \ self.path(X, y[:, k], l1_ratio=self.l1_ratio, eps=None, - n_alphas=None, alphas=[self.alpha], + n_alphas=None, alphas=[alpha], precompute=precompute, Xy=this_Xy, fit_intercept=False, normalize=False, copy_X=True, verbose=False, tol=self.tol, positive=self.positive, @@ -783,7 +870,7 @@ def _decision_function(self, X): Returns ------- - T : array, shape (n_samples,) + T : ndarray of shape (n_samples,) The predicted decision function """ check_is_fitted(self) @@ -811,19 +898,19 @@ class Lasso(ElasticNet): Parameters ---------- - alpha : float, optional + alpha : float, default=1.0 Constant that multiplies the L1 term. Defaults to 1.0. ``alpha = 0`` is equivalent to an ordinary least square, solved by the :class:`LinearRegression` object. For numerical reasons, using ``alpha = 0`` with the ``Lasso`` object is not advised. Given this, you should use the :class:`LinearRegression` object. - fit_intercept : boolean, optional, default True + fit_intercept : bool, default=True Whether to calculate the intercept for this model. If set to False, no intercept will be used in calculations (i.e. data is expected to be centered). - normalize : boolean, optional, default False + normalize : bool, default=False This parameter is ignored when ``fit_intercept`` is set to False. If True, the regressors X will be normalized before regression by subtracting the mean and dividing by the l2-norm. @@ -831,41 +918,40 @@ class Lasso(ElasticNet): :class:`sklearn.preprocessing.StandardScaler` before calling ``fit`` on an estimator with ``normalize=False``. - precompute : True | False | array-like, default=False + precompute : 'auto', bool or array-like of shape (n_features, n_features),\ + default=False Whether to use a precomputed Gram matrix to speed up calculations. If set to ``'auto'`` let us decide. The Gram matrix can also be passed as argument. For sparse input this option is always ``True`` to preserve sparsity. - copy_X : boolean, optional, default True + copy_X : bool, default=True If ``True``, X will be copied; else, it may be overwritten. - max_iter : int, optional + max_iter : int, default=1000 The maximum number of iterations - tol : float, optional + tol : float, default=1e-4 The tolerance for the optimization: if the updates are smaller than ``tol``, the optimization code checks the dual gap for optimality and continues until it is smaller than ``tol``. - warm_start : bool, optional + warm_start : bool, default=False When set to True, reuse the solution of the previous call to fit as initialization, otherwise, just erase the previous solution. See :term:`the Glossary `. - positive : bool, optional + positive : bool, default=False When set to ``True``, forces the coefficients to be positive. - random_state : int, RandomState instance or None, optional, default None + random_state : int, RandomState instance, default=None The seed of the pseudo random number generator that selects a random - feature to update. If int, random_state is the seed used by the random - number generator; If RandomState instance, random_state is the random - number generator; If None, the random number generator is the - RandomState instance used by `np.random`. Used when ``selection`` == - 'random'. + feature to update. Used when ``selection`` == 'random'. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. - selection : str, default 'cyclic' + selection : {'cyclic', 'random'}, default='cyclic' If set to 'random', a random coefficient is updated every iteration rather than looping over features sequentially by default. This (setting to 'random') often leads to significantly faster convergence @@ -873,17 +959,17 @@ class Lasso(ElasticNet): Attributes ---------- - coef_ : array, shape (n_features,) | (n_targets, n_features) + coef_ : ndarray of shape (n_features,) or (n_targets, n_features) parameter vector (w in the cost function formula) - sparse_coef_ : scipy.sparse matrix, shape (n_features, 1) | \ + sparse_coef_ : sparse matrix of shape (n_features, 1) or \ (n_targets, n_features) ``sparse_coef_`` is a readonly property derived from ``coef_`` - intercept_ : float | array, shape (n_targets,) + intercept_ : float or ndarray of shape (n_targets,) independent term in decision function. - n_iter_ : int | array-like, shape (n_targets,) + n_iter_ : int or list of int number of iterations run by the coordinate descent solver to reach the specified tolerance. @@ -937,10 +1023,10 @@ def _path_residuals(X, y, train, test, path, path_params, alphas=None, Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data. - y : array-like, shape (n_samples,) or (n_samples, n_targets) + y : array-like of shape (n_samples,) or (n_samples, n_targets) Target values train : list of indices @@ -956,21 +1042,21 @@ def _path_residuals(X, y, train, test, path, path_params, alphas=None, path_params : dictionary Parameters passed to the path function - alphas : array-like, optional + alphas : array-like, default=None Array of float that is used for cross-validation. If not provided, computed using 'path'. - l1_ratio : float, optional + l1_ratio : float, default=1 float between 0 and 1 passed to ElasticNet (scaling between l1 and l2 penalties). For ``l1_ratio = 0`` the penalty is an L2 penalty. For ``l1_ratio = 1`` it is an L1 penalty. For ``0 < l1_ratio < 1``, the penalty is a combination of L1 and L2. - X_order : {'F', 'C', or None}, optional + X_order : {'F', 'C'}, default=None The order of the arrays expected by the path function to avoid memory copies - dtype : a numpy dtype or None + dtype : a numpy dtype, default=None The dtype of the arrays expected by the path function to avoid memory copies """ @@ -1059,12 +1145,12 @@ def fit(self, X, y): Parameters ---------- - X : {array-like}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data. Pass directly as Fortran-contiguous data to avoid unnecessary memory duplication. If y is mono-output, X can be sparse. - y : array-like, shape (n_samples,) or (n_samples, n_targets) + y : array-like of shape (n_samples,) or (n_samples, n_targets) Target values """ y = check_array(y, copy=False, dtype=[np.float64, np.float32], @@ -1113,8 +1199,8 @@ def fit(self, X, y): # Let us not impose fortran ordering so far: it is # not useful for the cross-validation loop and will be done # by the model fitting itself - X = check_array(X, 'csc', dtype=[np.float64, np.float32], - copy=False) + X = self._validate_data(X, accept_sparse='csc', + dtype=[np.float64, np.float32], copy=False) if sparse.isspmatrix(X): if (hasattr(reference_to_old_X, "data") and not np.may_share_memory(reference_to_old_X.data, X.data)): @@ -1125,8 +1211,9 @@ def fit(self, X, y): copy_X = False del reference_to_old_X else: - X = check_array(X, 'csc', dtype=[np.float64, np.float32], - order='F', copy=copy_X) + X = self._validate_data(X, accept_sparse='csc', + dtype=[np.float64, np.float32], order='F', + copy=copy_X) copy_X = False if X.shape[0] != y.shape[0]: @@ -1240,23 +1327,23 @@ class LassoCV(RegressorMixin, LinearModelCV): Parameters ---------- - eps : float, optional + eps : float, default=1e-3 Length of the path. ``eps=1e-3`` means that ``alpha_min / alpha_max = 1e-3``. - n_alphas : int, optional + n_alphas : int, default=100 Number of alphas along the regularization path - alphas : numpy array, optional + alphas : ndarray, default=None List of alphas where to compute the models. If ``None`` alphas are set automatically - fit_intercept : boolean, default True + fit_intercept : bool, default=True whether to calculate the intercept for this model. If set to false, no intercept will be used in calculations (i.e. data is expected to be centered). - normalize : boolean, optional, default False + normalize : bool, default=False This parameter is ignored when ``fit_intercept`` is set to False. If True, the regressors X will be normalized before regression by subtracting the mean and dividing by the l2-norm. @@ -1264,33 +1351,34 @@ class LassoCV(RegressorMixin, LinearModelCV): :class:`sklearn.preprocessing.StandardScaler` before calling ``fit`` on an estimator with ``normalize=False``. - precompute : True | False | 'auto' | array-like + precompute : 'auto', bool or array-like of shape (n_features, n_features),\ + default='auto' Whether to use a precomputed Gram matrix to speed up calculations. If set to ``'auto'`` let us decide. The Gram matrix can also be passed as argument. - max_iter : int, optional + max_iter : int, default=1000 The maximum number of iterations - tol : float, optional + tol : float, default=1e-4 The tolerance for the optimization: if the updates are smaller than ``tol``, the optimization code checks the dual gap for optimality and continues until it is smaller than ``tol``. - copy_X : boolean, optional, default True + copy_X : bool, default=True If ``True``, X will be copied; else, it may be overwritten. - cv : int, cross-validation generator or an iterable, optional + cv : int, cross-validation generator or iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross-validation, - - integer, to specify the number of folds. + - int, to specify the number of folds. - :term:`CV splitter`, - An iterable yielding (train, test) splits as arrays of indices. - For integer/None inputs, :class:`KFold` is used. + For int/None inputs, :class:`KFold` is used. Refer :ref:`User Guide ` for the various cross-validation strategies that can be used here. @@ -1298,27 +1386,25 @@ class LassoCV(RegressorMixin, LinearModelCV): .. versionchanged:: 0.22 ``cv`` default value if None changed from 3-fold to 5-fold. - verbose : bool or integer + verbose : bool or int, default=False Amount of verbosity. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None Number of CPUs to use during the cross validation. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - positive : bool, optional + positive : bool, default=False If positive, restrict regression coefficients to be positive - random_state : int, RandomState instance or None, optional, default None + random_state : int, RandomState instance, default=None The seed of the pseudo random number generator that selects a random - feature to update. If int, random_state is the seed used by the random - number generator; If RandomState instance, random_state is the random - number generator; If None, the random number generator is the - RandomState instance used by `np.random`. Used when ``selection`` == - 'random'. + feature to update. Used when ``selection`` == 'random'. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. - selection : str, default 'cyclic' + selection : {'cyclic', 'random'}, default='cyclic' If set to 'random', a random coefficient is updated every iteration rather than looping over features sequentially by default. This (setting to 'random') often leads to significantly faster convergence @@ -1329,19 +1415,19 @@ class LassoCV(RegressorMixin, LinearModelCV): alpha_ : float The amount of penalization chosen by cross validation - coef_ : array, shape (n_features,) | (n_targets, n_features) + coef_ : ndarray of shape (n_features,) or (n_targets, n_features) parameter vector (w in the cost function formula) - intercept_ : float | array, shape (n_targets,) + intercept_ : float or ndarray of shape (n_targets,) independent term in decision function. - mse_path_ : array, shape (n_alphas, n_folds) + mse_path_ : ndarray of shape (n_alphas, n_folds) mean square error for the test set on each fold, varying alpha - alphas_ : numpy array, shape (n_alphas,) + alphas_ : ndarray of shape (n_alphas,) The grid of alphas used for fitting - dual_gap_ : ndarray, shape () + dual_gap_ : float or ndarray of shape (n_targets,) The dual gap at the end of the optimization for the optimal alpha (``alpha_``). @@ -1393,6 +1479,7 @@ def __init__(self, eps=1e-3, n_alphas=100, alphas=None, fit_intercept=True, def _more_tags(self): return {'multioutput': False} + class ElasticNetCV(RegressorMixin, LinearModelCV): """Elastic Net model with iterative fitting along a regularization path. @@ -1402,7 +1489,7 @@ class ElasticNetCV(RegressorMixin, LinearModelCV): Parameters ---------- - l1_ratio : float or array of floats, optional + l1_ratio : float or list of float, default=0.5 float between 0 and 1 passed to ElasticNet (scaling between l1 and l2 penalties). For ``l1_ratio = 0`` the penalty is an L2 penalty. For ``l1_ratio = 1`` it is an L1 penalty. @@ -1414,23 +1501,23 @@ class ElasticNetCV(RegressorMixin, LinearModelCV): (i.e. Lasso) and less close to 0 (i.e. Ridge), as in ``[.1, .5, .7, .9, .95, .99, 1]`` - eps : float, optional + eps : float, default=1e-3 Length of the path. ``eps=1e-3`` means that ``alpha_min / alpha_max = 1e-3``. - n_alphas : int, optional + n_alphas : int, default=100 Number of alphas along the regularization path, used for each l1_ratio. - alphas : numpy array, optional + alphas : ndarray, default=None List of alphas where to compute the models. If None alphas are set automatically - fit_intercept : boolean + fit_intercept : bool, default=True whether to calculate the intercept for this model. If set to false, no intercept will be used in calculations (i.e. data is expected to be centered). - normalize : boolean, optional, default False + normalize : bool, default=False This parameter is ignored when ``fit_intercept`` is set to False. If True, the regressors X will be normalized before regression by subtracting the mean and dividing by the l2-norm. @@ -1438,30 +1525,31 @@ class ElasticNetCV(RegressorMixin, LinearModelCV): :class:`sklearn.preprocessing.StandardScaler` before calling ``fit`` on an estimator with ``normalize=False``. - precompute : True | False | 'auto' | array-like + precompute : 'auto', bool or array-like of shape (n_features, n_features),\ + default='auto' Whether to use a precomputed Gram matrix to speed up calculations. If set to ``'auto'`` let us decide. The Gram matrix can also be passed as argument. - max_iter : int, optional + max_iter : int, default=1000 The maximum number of iterations - tol : float, optional + tol : float, default=1e-4 The tolerance for the optimization: if the updates are smaller than ``tol``, the optimization code checks the dual gap for optimality and continues until it is smaller than ``tol``. - cv : int, cross-validation generator or an iterable, optional + cv : int, cross-validation generator or iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross-validation, - - integer, to specify the number of folds. + - int, to specify the number of folds. - :term:`CV splitter`, - An iterable yielding (train, test) splits as arrays of indices. - For integer/None inputs, :class:`KFold` is used. + For int/None inputs, :class:`KFold` is used. Refer :ref:`User Guide ` for the various cross-validation strategies that can be used here. @@ -1469,30 +1557,28 @@ class ElasticNetCV(RegressorMixin, LinearModelCV): .. versionchanged:: 0.22 ``cv`` default value if None changed from 3-fold to 5-fold. - copy_X : boolean, optional, default True + copy_X : bool, default=True If ``True``, X will be copied; else, it may be overwritten. - verbose : bool or integer + verbose : bool or int, default=0 Amount of verbosity. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None Number of CPUs to use during the cross validation. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - positive : bool, optional + positive : bool, default=False When set to ``True``, forces the coefficients to be positive. - random_state : int, RandomState instance or None, optional, default None + random_state : int, RandomState instance, default=None The seed of the pseudo random number generator that selects a random - feature to update. If int, random_state is the seed used by the random - number generator; If RandomState instance, random_state is the random - number generator; If None, the random number generator is the - RandomState instance used by `np.random`. Used when ``selection`` == - 'random'. + feature to update. Used when ``selection`` == 'random'. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. - selection : str, default 'cyclic' + selection : {'cyclic', 'random'}, default='cyclic' If set to 'random', a random coefficient is updated every iteration rather than looping over features sequentially by default. This (setting to 'random') often leads to significantly faster convergence @@ -1507,17 +1593,17 @@ class ElasticNetCV(RegressorMixin, LinearModelCV): The compromise between l1 and l2 penalization chosen by cross validation - coef_ : array, shape (n_features,) | (n_targets, n_features) + coef_ : ndarray of shape (n_features,) or (n_targets, n_features) Parameter vector (w in the cost function formula), - intercept_ : float | array, shape (n_targets, n_features) + intercept_ : float or ndarray of shape (n_targets, n_features) Independent term in the decision function. - mse_path_ : array, shape (n_l1_ratio, n_alpha, n_folds) + mse_path_ : ndarray of shape (n_l1_ratio, n_alpha, n_folds) Mean square error for the test set on each fold, varying l1_ratio and alpha. - alphas_ : numpy array, shape (n_alphas,) or (n_l1_ratio, n_alphas) + alphas_ : ndarray of shape (n_alphas,) or (n_l1_ratio, n_alphas) The grid of alphas used for fitting, for each l1_ratio. n_iter_ : int @@ -1623,21 +1709,21 @@ class MultiTaskElasticNet(Lasso): Parameters ---------- - alpha : float, optional + alpha : float, default=1.0 Constant that multiplies the L1/L2 term. Defaults to 1.0 - l1_ratio : float + l1_ratio : float, default=0.5 The ElasticNet mixing parameter, with 0 < l1_ratio <= 1. For l1_ratio = 1 the penalty is an L1/L2 penalty. For l1_ratio = 0 it is an L2 penalty. For ``0 < l1_ratio < 1``, the penalty is a combination of L1/L2 and L2. - fit_intercept : boolean + fit_intercept : bool, default=True whether to calculate the intercept for this model. If set to false, no intercept will be used in calculations (i.e. data is expected to be centered). - normalize : boolean, optional, default False + normalize : bool, default=False This parameter is ignored when ``fit_intercept`` is set to False. If True, the regressors X will be normalized before regression by subtracting the mean and dividing by the l2-norm. @@ -1645,32 +1731,30 @@ class MultiTaskElasticNet(Lasso): :class:`sklearn.preprocessing.StandardScaler` before calling ``fit`` on an estimator with ``normalize=False``. - copy_X : boolean, optional, default True + copy_X : bool, default=True If ``True``, X will be copied; else, it may be overwritten. - max_iter : int, optional + max_iter : int, default=1000 The maximum number of iterations - tol : float, optional + tol : float, default=1e-4 The tolerance for the optimization: if the updates are smaller than ``tol``, the optimization code checks the dual gap for optimality and continues until it is smaller than ``tol``. - warm_start : bool, optional + warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit as initialization, otherwise, just erase the previous solution. See :term:`the Glossary `. - random_state : int, RandomState instance or None, optional, default None + random_state : int, RandomState instance, default=None The seed of the pseudo random number generator that selects a random - feature to update. If int, random_state is the seed used by the random - number generator; If RandomState instance, random_state is the random - number generator; If None, the random number generator is the - RandomState instance used by `np.random`. Used when ``selection`` == - 'random'. + feature to update. Used when ``selection`` == 'random'. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. - selection : str, default 'cyclic' + selection : {'cyclic', 'random'}, default='cyclic' If set to 'random', a random coefficient is updated every iteration rather than looping over features sequentially by default. This (setting to 'random') often leads to significantly faster convergence @@ -1678,10 +1762,10 @@ class MultiTaskElasticNet(Lasso): Attributes ---------- - intercept_ : array, shape (n_tasks,) + intercept_ : ndarray of shape (n_tasks,) Independent term in decision function. - coef_ : array, shape (n_tasks, n_features) + coef_ : ndarray of shape (n_tasks, n_features) Parameter vector (W in the cost function formula). If a 1D y is passed in at fit (non multi-task usage), ``coef_`` is then a 1D array. Note that ``coef_`` stores the transpose of ``W``, ``W.T``. @@ -1735,9 +1819,9 @@ def fit(self, X, y): Parameters ---------- - X : ndarray, shape (n_samples, n_features) + X : ndarray of shape (n_samples, n_features) Data - y : ndarray, shape (n_samples, n_tasks) + y : ndarray of shape (n_samples, n_tasks) Target. Will be cast to X's dtype if necessary Notes @@ -1750,8 +1834,8 @@ def fit(self, X, y): To avoid memory re-allocation it is advised to allocate the initial data in memory directly using that format. """ - X = check_array(X, dtype=[np.float64, np.float32], order='F', - copy=self.copy_X and self.fit_intercept) + X = self._validate_data(X, dtype=[np.float64, np.float32], order='F', + copy=self.copy_X and self.fit_intercept) y = check_array(y, dtype=X.dtype.type, ensure_2d=False) if hasattr(self, 'l1_ratio'): @@ -1815,15 +1899,15 @@ class MultiTaskLasso(MultiTaskElasticNet): Parameters ---------- - alpha : float, optional + alpha : float, default=1.0 Constant that multiplies the L1/L2 term. Defaults to 1.0 - fit_intercept : boolean + fit_intercept : bool, default=True whether to calculate the intercept for this model. If set to false, no intercept will be used in calculations (i.e. data is expected to be centered). - normalize : boolean, optional, default False + normalize : bool, default=False This parameter is ignored when ``fit_intercept`` is set to False. If True, the regressors X will be normalized before regression by subtracting the mean and dividing by the l2-norm. @@ -1831,32 +1915,30 @@ class MultiTaskLasso(MultiTaskElasticNet): :class:`sklearn.preprocessing.StandardScaler` before calling ``fit`` on an estimator with ``normalize=False``. - copy_X : boolean, optional, default True + copy_X : bool, default=True If ``True``, X will be copied; else, it may be overwritten. - max_iter : int, optional + max_iter : int, default=1000 The maximum number of iterations - tol : float, optional + tol : float, default=1e-4 The tolerance for the optimization: if the updates are smaller than ``tol``, the optimization code checks the dual gap for optimality and continues until it is smaller than ``tol``. - warm_start : bool, optional + warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit as initialization, otherwise, just erase the previous solution. See :term:`the Glossary `. - random_state : int, RandomState instance or None, optional, default None + random_state : int, RandomState instance, default=None The seed of the pseudo random number generator that selects a random - feature to update. If int, random_state is the seed used by the random - number generator; If RandomState instance, random_state is the random - number generator; If None, the random number generator is the - RandomState instance used by `np.random`. Used when ``selection`` == - 'random'. + feature to update. Used when ``selection`` == 'random'. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. - selection : str, default 'cyclic' + selection : {'cyclic', 'random'}, default='cyclic' If set to 'random', a random coefficient is updated every iteration rather than looping over features sequentially by default. This (setting to 'random') often leads to significantly faster convergence @@ -1864,11 +1946,11 @@ class MultiTaskLasso(MultiTaskElasticNet): Attributes ---------- - coef_ : array, shape (n_tasks, n_features) + coef_ : ndarray of shape (n_tasks, n_features) Parameter vector (W in the cost function formula). Note that ``coef_`` stores the transpose of ``W``, ``W.T``. - intercept_ : array, shape (n_tasks,) + intercept_ : ndarray of shape (n_tasks,) independent term in decision function. n_iter_ : int @@ -1938,7 +2020,7 @@ class MultiTaskElasticNetCV(RegressorMixin, LinearModelCV): Parameters ---------- - l1_ratio : float or array of floats + l1_ratio : float or list of float, default=0.5 The ElasticNet mixing parameter, with 0 < l1_ratio <= 1. For l1_ratio = 1 the penalty is an L1/L2 penalty. For l1_ratio = 0 it is an L2 penalty. @@ -1950,23 +2032,23 @@ class MultiTaskElasticNetCV(RegressorMixin, LinearModelCV): (i.e. Lasso) and less close to 0 (i.e. Ridge), as in ``[.1, .5, .7, .9, .95, .99, 1]`` - eps : float, optional + eps : float, default=1e-3 Length of the path. ``eps=1e-3`` means that ``alpha_min / alpha_max = 1e-3``. - n_alphas : int, optional + n_alphas : int, default=100 Number of alphas along the regularization path - alphas : array-like, optional + alphas : array-like, default=None List of alphas where to compute the models. If not provided, set automatically. - fit_intercept : boolean + fit_intercept : bool, default=True whether to calculate the intercept for this model. If set to false, no intercept will be used in calculations (i.e. data is expected to be centered). - normalize : boolean, optional, default False + normalize : bool, default=False This parameter is ignored when ``fit_intercept`` is set to False. If True, the regressors X will be normalized before regression by subtracting the mean and dividing by the l2-norm. @@ -1974,25 +2056,25 @@ class MultiTaskElasticNetCV(RegressorMixin, LinearModelCV): :class:`sklearn.preprocessing.StandardScaler` before calling ``fit`` on an estimator with ``normalize=False``. - max_iter : int, optional + max_iter : int, default=1000 The maximum number of iterations - tol : float, optional + tol : float, default=1e-4 The tolerance for the optimization: if the updates are smaller than ``tol``, the optimization code checks the dual gap for optimality and continues until it is smaller than ``tol``. - cv : int, cross-validation generator or an iterable, optional + cv : int, cross-validation generator or iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross-validation, - - integer, to specify the number of folds. + - int, to specify the number of folds. - :term:`CV splitter`, - An iterable yielding (train, test) splits as arrays of indices. - For integer/None inputs, :class:`KFold` is used. + For int/None inputs, :class:`KFold` is used. Refer :ref:`User Guide ` for the various cross-validation strategies that can be used here. @@ -2000,28 +2082,26 @@ class MultiTaskElasticNetCV(RegressorMixin, LinearModelCV): .. versionchanged:: 0.22 ``cv`` default value if None changed from 3-fold to 5-fold. - copy_X : boolean, optional, default True + copy_X : bool, default=True If ``True``, X will be copied; else, it may be overwritten. - verbose : bool or integer + verbose : bool or int, default=0 Amount of verbosity. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None Number of CPUs to use during the cross validation. Note that this is used only if multiple values for l1_ratio are given. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance or None, optional, default None + random_state : int, RandomState instance, default=None The seed of the pseudo random number generator that selects a random - feature to update. If int, random_state is the seed used by the random - number generator; If RandomState instance, random_state is the random - number generator; If None, the random number generator is the - RandomState instance used by `np.random`. Used when ``selection`` == - 'random'. + feature to update. Used when ``selection`` == 'random'. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. - selection : str, default 'cyclic' + selection : {'cyclic', 'random'}, default='cyclic' If set to 'random', a random coefficient is updated every iteration rather than looping over features sequentially by default. This (setting to 'random') often leads to significantly faster convergence @@ -2029,21 +2109,21 @@ class MultiTaskElasticNetCV(RegressorMixin, LinearModelCV): Attributes ---------- - intercept_ : array, shape (n_tasks,) + intercept_ : ndarray of shape (n_tasks,) Independent term in decision function. - coef_ : array, shape (n_tasks, n_features) + coef_ : ndarray of shape (n_tasks, n_features) Parameter vector (W in the cost function formula). Note that ``coef_`` stores the transpose of ``W``, ``W.T``. alpha_ : float The amount of penalization chosen by cross validation - mse_path_ : array, shape (n_alphas, n_folds) or \ + mse_path_ : ndarray of shape (n_alphas, n_folds) or \ (n_l1_ratio, n_alphas, n_folds) mean square error for the test set on each fold, varying alpha - alphas_ : numpy array, shape (n_alphas,) or (n_l1_ratio, n_alphas) + alphas_ : ndarray of shape (n_alphas,) or (n_l1_ratio, n_alphas) The grid of alphas used for fitting, for each l1_ratio l1_ratio_ : float @@ -2126,23 +2206,23 @@ class MultiTaskLassoCV(RegressorMixin, LinearModelCV): Parameters ---------- - eps : float, optional + eps : float, default=1e-3 Length of the path. ``eps=1e-3`` means that ``alpha_min / alpha_max = 1e-3``. - n_alphas : int, optional + n_alphas : int, default=100 Number of alphas along the regularization path - alphas : array-like, optional + alphas : array-like, default=None List of alphas where to compute the models. If not provided, set automatically. - fit_intercept : boolean + fit_intercept : bool, default=True whether to calculate the intercept for this model. If set to false, no intercept will be used in calculations (i.e. data is expected to be centered). - normalize : boolean, optional, default False + normalize : bool, default=False This parameter is ignored when ``fit_intercept`` is set to False. If True, the regressors X will be normalized before regression by subtracting the mean and dividing by the l2-norm. @@ -2150,28 +2230,28 @@ class MultiTaskLassoCV(RegressorMixin, LinearModelCV): :class:`sklearn.preprocessing.StandardScaler` before calling ``fit`` on an estimator with ``normalize=False``. - max_iter : int, optional + max_iter : int, default=1000 The maximum number of iterations. - tol : float, optional + tol : float, default=1e-4 The tolerance for the optimization: if the updates are smaller than ``tol``, the optimization code checks the dual gap for optimality and continues until it is smaller than ``tol``. - copy_X : boolean, optional, default True + copy_X : bool, default=True If ``True``, X will be copied; else, it may be overwritten. - cv : int, cross-validation generator or an iterable, optional + cv : int, cross-validation generator or iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross-validation, - - integer, to specify the number of folds. + - int, to specify the number of folds. - :term:`CV splitter`, - An iterable yielding (train, test) splits as arrays of indices. - For integer/None inputs, :class:`KFold` is used. + For int/None inputs, :class:`KFold` is used. Refer :ref:`User Guide ` for the various cross-validation strategies that can be used here. @@ -2179,25 +2259,23 @@ class MultiTaskLassoCV(RegressorMixin, LinearModelCV): .. versionchanged:: 0.22 ``cv`` default value if None changed from 3-fold to 5-fold. - verbose : bool or integer + verbose : bool or int, default=False Amount of verbosity. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None Number of CPUs to use during the cross validation. Note that this is used only if multiple values for l1_ratio are given. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance or None, optional, default None + random_state : int, RandomState instance, default=None The seed of the pseudo random number generator that selects a random - feature to update. If int, random_state is the seed used by the random - number generator; If RandomState instance, random_state is the random - number generator; If None, the random number generator is the - RandomState instance used by `np.random`. Used when ``selection`` == - 'random' + feature to update. Used when ``selection`` == 'random'. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. - selection : str, default 'cyclic' + selection : {'cyclic', 'random'}, default='cyclic' If set to 'random', a random coefficient is updated every iteration rather than looping over features sequentially by default. This (setting to 'random') often leads to significantly faster convergence @@ -2205,20 +2283,20 @@ class MultiTaskLassoCV(RegressorMixin, LinearModelCV): Attributes ---------- - intercept_ : array, shape (n_tasks,) + intercept_ : ndarray of shape (n_tasks,) Independent term in decision function. - coef_ : array, shape (n_tasks, n_features) + coef_ : ndarray of shape (n_tasks, n_features) Parameter vector (W in the cost function formula). Note that ``coef_`` stores the transpose of ``W``, ``W.T``. alpha_ : float The amount of penalization chosen by cross validation - mse_path_ : array, shape (n_alphas, n_folds) + mse_path_ : ndarray of shape (n_alphas, n_folds) mean square error for the test set on each fold, varying alpha - alphas_ : numpy array, shape (n_alphas,) + alphas_ : ndarray of shape (n_alphas,) The grid of alphas used for fitting. n_iter_ : int diff --git a/sklearn/linear_model/_glm/__init__.py b/sklearn/linear_model/_glm/__init__.py new file mode 100644 index 0000000000000..3b5c0d95d6124 --- /dev/null +++ b/sklearn/linear_model/_glm/__init__.py @@ -0,0 +1,15 @@ +# License: BSD 3 clause + +from .glm import ( + GeneralizedLinearRegressor, + PoissonRegressor, + GammaRegressor, + TweedieRegressor +) + +__all__ = [ + "GeneralizedLinearRegressor", + "PoissonRegressor", + "GammaRegressor", + "TweedieRegressor" +] diff --git a/sklearn/linear_model/_glm/glm.py b/sklearn/linear_model/_glm/glm.py new file mode 100644 index 0000000000000..8607d6a1828ab --- /dev/null +++ b/sklearn/linear_model/_glm/glm.py @@ -0,0 +1,615 @@ +""" +Generalized Linear Models with Exponential Dispersion Family +""" + +# Author: Christian Lorentzen +# some parts and tricks stolen from other sklearn files. +# License: BSD 3 clause + +import numbers + +import numpy as np +import scipy.optimize + +from ...base import BaseEstimator, RegressorMixin +from ...utils import check_array, check_X_y +from ...utils.optimize import _check_optimize_result +from ...utils.validation import check_is_fitted, _check_sample_weight +from ..._loss.glm_distribution import ( + ExponentialDispersionModel, + TweedieDistribution, + EDM_DISTRIBUTIONS +) +from .link import ( + BaseLink, + IdentityLink, + LogLink, +) + + +def _safe_lin_pred(X, coef): + """Compute the linear predictor taking care if intercept is present.""" + if coef.size == X.shape[1] + 1: + return X @ coef[1:] + coef[0] + else: + return X @ coef + + +def _y_pred_deviance_derivative(coef, X, y, weights, family, link): + """Compute y_pred and the derivative of the deviance w.r.t coef.""" + lin_pred = _safe_lin_pred(X, coef) + y_pred = link.inverse(lin_pred) + d1 = link.inverse_derivative(lin_pred) + temp = d1 * family.deviance_derivative(y, y_pred, weights) + if coef.size == X.shape[1] + 1: + devp = np.concatenate(([temp.sum()], temp @ X)) + else: + devp = temp @ X # same as X.T @ temp + return y_pred, devp + + +class GeneralizedLinearRegressor(BaseEstimator, RegressorMixin): + """Regression via a penalized Generalized Linear Model (GLM). + + GLMs based on a reproductive Exponential Dispersion Model (EDM) aim at + fitting and predicting the mean of the target y as y_pred=h(X*w). + Therefore, the fit minimizes the following objective function with L2 + priors as regularizer:: + + 1/(2*sum(s)) * deviance(y, h(X*w); s) + + 1/2 * alpha * |w|_2 + + with inverse link function h and s=sample_weight. + The parameter ``alpha`` corresponds to the lambda parameter in glmnet. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + alpha : float, default=1 + Constant that multiplies the penalty term and thus determines the + regularization strength. ``alpha = 0`` is equivalent to unpenalized + GLMs. In this case, the design matrix `X` must have full column rank + (no collinearities). + + fit_intercept : bool, default=True + Specifies if a constant (a.k.a. bias or intercept) should be + added to the linear predictor (X @ coef + intercept). + + family : {'normal', 'poisson', 'gamma', 'inverse-gaussian'} \ + or an ExponentialDispersionModel instance, default='normal' + The distributional assumption of the GLM, i.e. which distribution from + the EDM, specifies the loss function to be minimized. + + link : {'auto', 'identity', 'log'} or an instance of class BaseLink, \ + default='auto' + The link function of the GLM, i.e. mapping from linear predictor + `X @ coeff + intercept` to prediction `y_pred`. Option 'auto' sets + the link depending on the chosen family as follows: + + - 'identity' for Normal distribution + - 'log' for Poisson, Gamma and Inverse Gaussian distributions + + solver : 'lbfgs', default='lbfgs' + Algorithm to use in the optimization problem: + + 'lbfgs' + Calls scipy's L-BFGS-B optimizer. + + max_iter : int, default=100 + The maximal number of iterations for the solver. + + tol : float, default=1e-4 + Stopping criterion. For the lbfgs solver, + the iteration will stop when ``max{|g_j|, j = 1, ..., d} <= tol`` + where ``g_j`` is the j-th component of the gradient (derivative) of + the objective function. + + warm_start : bool, default=False + If set to ``True``, reuse the solution of the previous call to ``fit`` + as initialization for ``coef_`` and ``intercept_``. + + verbose : int, default=0 + For the lbfgs solver set verbose to any positive number for verbosity. + + Attributes + ---------- + coef_ : array of shape (n_features,) + Estimated coefficients for the linear predictor (`X @ coef_ + + intercept_`) in the GLM. + + intercept_ : float + Intercept (a.k.a. bias) added to linear predictor. + + n_iter_ : int + Actual number of iterations used in the solver. + """ + def __init__(self, *, alpha=1.0, + fit_intercept=True, family='normal', link='auto', + solver='lbfgs', max_iter=100, tol=1e-4, warm_start=False, + verbose=0): + self.alpha = alpha + self.fit_intercept = fit_intercept + self.family = family + self.link = link + self.solver = solver + self.max_iter = max_iter + self.tol = tol + self.warm_start = warm_start + self.verbose = verbose + + def fit(self, X, y, sample_weight=None): + """Fit a Generalized Linear Model. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training data. + + y : array-like of shape (n_samples,) + Target values. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + + Returns + ------- + self : returns an instance of self. + """ + if isinstance(self.family, ExponentialDispersionModel): + self._family_instance = self.family + elif self.family in EDM_DISTRIBUTIONS: + self._family_instance = EDM_DISTRIBUTIONS[self.family]() + else: + raise ValueError( + "The family must be an instance of class" + " ExponentialDispersionModel or an element of" + " ['normal', 'poisson', 'gamma', 'inverse-gaussian']" + "; got (family={0})".format(self.family)) + + # Guarantee that self._link_instance is set to an instance of + # class BaseLink + if isinstance(self.link, BaseLink): + self._link_instance = self.link + else: + if self.link == 'auto': + if isinstance(self._family_instance, TweedieDistribution): + if self._family_instance.power <= 0: + self._link_instance = IdentityLink() + if self._family_instance.power >= 1: + self._link_instance = LogLink() + else: + raise ValueError("No default link known for the " + "specified distribution family. Please " + "set link manually, i.e. not to 'auto'; " + "got (link='auto', family={})" + .format(self.family)) + elif self.link == 'identity': + self._link_instance = IdentityLink() + elif self.link == 'log': + self._link_instance = LogLink() + else: + raise ValueError( + "The link must be an instance of class Link or " + "an element of ['auto', 'identity', 'log']; " + "got (link={0})".format(self.link)) + + if not isinstance(self.alpha, numbers.Number) or self.alpha < 0: + raise ValueError("Penalty term must be a non-negative number;" + " got (alpha={0})".format(self.alpha)) + if not isinstance(self.fit_intercept, bool): + raise ValueError("The argument fit_intercept must be bool;" + " got {0}".format(self.fit_intercept)) + if self.solver not in ['lbfgs']: + raise ValueError("GeneralizedLinearRegressor supports only solvers" + "'lbfgs'; got {0}".format(self.solver)) + solver = self.solver + if (not isinstance(self.max_iter, numbers.Integral) + or self.max_iter <= 0): + raise ValueError("Maximum number of iteration must be a positive " + "integer;" + " got (max_iter={0!r})".format(self.max_iter)) + if not isinstance(self.tol, numbers.Number) or self.tol <= 0: + raise ValueError("Tolerance for stopping criteria must be " + "positive; got (tol={0!r})".format(self.tol)) + if not isinstance(self.warm_start, bool): + raise ValueError("The argument warm_start must be bool;" + " got {0}".format(self.warm_start)) + + family = self._family_instance + link = self._link_instance + + X, y = check_X_y(X, y, accept_sparse=['csc', 'csr'], + dtype=[np.float64, np.float32], + y_numeric=True, multi_output=False) + + weights = _check_sample_weight(sample_weight, X) + + _, n_features = X.shape + + if not np.all(family.in_y_range(y)): + raise ValueError("Some value(s) of y are out of the valid " + "range for family {0}" + .format(family.__class__.__name__)) + # TODO: if alpha=0 check that X is not rank deficient + + # rescaling of sample_weight + # + # IMPORTANT NOTE: Since we want to minimize + # 1/(2*sum(sample_weight)) * deviance + L2, + # deviance = sum(sample_weight * unit_deviance), + # we rescale weights such that sum(weights) = 1 and this becomes + # 1/2*deviance + L2 with deviance=sum(weights * unit_deviance) + weights = weights / weights.sum() + + if self.warm_start and hasattr(self, 'coef_'): + if self.fit_intercept: + coef = np.concatenate((np.array([self.intercept_]), + self.coef_)) + else: + coef = self.coef_ + else: + if self.fit_intercept: + coef = np.zeros(n_features+1) + coef[0] = link(np.average(y, weights=weights)) + else: + coef = np.zeros(n_features) + + # algorithms for optimization + + if solver == 'lbfgs': + def func(coef, X, y, weights, alpha, family, link): + y_pred, devp = _y_pred_deviance_derivative( + coef, X, y, weights, family, link + ) + dev = family.deviance(y, y_pred, weights) + # offset if coef[0] is intercept + offset = 1 if self.fit_intercept else 0 + coef_scaled = alpha * coef[offset:] + obj = 0.5 * dev + 0.5 * (coef[offset:] @ coef_scaled) + objp = 0.5 * devp + objp[offset:] += coef_scaled + return obj, objp + + args = (X, y, weights, self.alpha, family, link) + + opt_res = scipy.optimize.minimize( + func, coef, method="L-BFGS-B", jac=True, + options={ + "maxiter": self.max_iter, + "iprint": (self.verbose > 0) - 1, + "gtol": self.tol, + "ftol": 1e3*np.finfo(float).eps, + }, + args=args) + self.n_iter_ = _check_optimize_result("lbfgs", opt_res) + coef = opt_res.x + + if self.fit_intercept: + self.intercept_ = coef[0] + self.coef_ = coef[1:] + else: + # set intercept to zero as the other linear models do + self.intercept_ = 0. + self.coef_ = coef + + return self + + def _linear_predictor(self, X): + """Compute the linear_predictor = `X @ coef_ + intercept_`. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Samples. + + Returns + ------- + y_pred : array of shape (n_samples,) + Returns predicted values of linear predictor. + """ + check_is_fitted(self) + X = check_array(X, accept_sparse=['csr', 'csc', 'coo'], + dtype=[np.float64, np.float32], ensure_2d=True, + allow_nd=False) + return X @ self.coef_ + self.intercept_ + + def predict(self, X): + """Predict using GLM with feature matrix X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Samples. + + Returns + ------- + y_pred : array of shape (n_samples,) + Returns predicted values. + """ + # check_array is done in _linear_predictor + eta = self._linear_predictor(X) + y_pred = self._link_instance.inverse(eta) + return y_pred + + def score(self, X, y, sample_weight=None): + """Compute D^2, the percentage of deviance explained. + + D^2 is a generalization of the coefficient of determination R^2. + R^2 uses squared error and D^2 deviance. Note that those two are equal + for ``family='normal'``. + + D^2 is defined as + :math:`D^2 = 1-\\frac{D(y_{true},y_{pred})}{D_{null}}`, + :math:`D_{null}` is the null deviance, i.e. the deviance of a model + with intercept alone, which corresponds to :math:`y_{pred} = \\bar{y}`. + The mean :math:`\\bar{y}` is averaged by sample_weight. + Best possible score is 1.0 and it can be negative (because the model + can be arbitrarily worse). + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Test samples. + + y : array-like of shape (n_samples,) + True values of target. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + + Returns + ------- + score : float + D^2 of self.predict(X) w.r.t. y. + """ + # Note, default score defined in RegressorMixin is R^2 score. + # TODO: make D^2 a score function in module metrics (and thereby get + # input validation and so on) + weights = _check_sample_weight(sample_weight, X) + y_pred = self.predict(X) + dev = self._family_instance.deviance(y, y_pred, weights=weights) + y_mean = np.average(y, weights=weights) + dev_null = self._family_instance.deviance(y, y_mean, weights=weights) + return 1 - dev / dev_null + + def _more_tags(self): + # create the _family_instance if fit wasn't called yet. + if hasattr(self, '_family_instance'): + _family_instance = self._family_instance + elif isinstance(self.family, ExponentialDispersionModel): + _family_instance = self.family + elif self.family in EDM_DISTRIBUTIONS: + _family_instance = EDM_DISTRIBUTIONS[self.family]() + else: + raise ValueError + return {"requires_positive_y": not _family_instance.in_y_range(-1.0)} + + +class PoissonRegressor(GeneralizedLinearRegressor): + """Generalized Linear Model with a Poisson distribution. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + alpha : float, default=1 + Constant that multiplies the penalty term and thus determines the + regularization strength. ``alpha = 0`` is equivalent to unpenalized + GLMs. In this case, the design matrix `X` must have full column rank + (no collinearities). + + fit_intercept : bool, default=True + Specifies if a constant (a.k.a. bias or intercept) should be + added to the linear predictor (X @ coef + intercept). + + max_iter : int, default=100 + The maximal number of iterations for the solver. + + tol : float, default=1e-4 + Stopping criterion. For the lbfgs solver, + the iteration will stop when ``max{|g_j|, j = 1, ..., d} <= tol`` + where ``g_j`` is the j-th component of the gradient (derivative) of + the objective function. + + warm_start : bool, default=False + If set to ``True``, reuse the solution of the previous call to ``fit`` + as initialization for ``coef_`` and ``intercept_`` . + + verbose : int, default=0 + For the lbfgs solver set verbose to any positive number for verbosity. + + Attributes + ---------- + coef_ : array of shape (n_features,) + Estimated coefficients for the linear predictor (`X @ coef_ + + intercept_`) in the GLM. + + intercept_ : float + Intercept (a.k.a. bias) added to linear predictor. + + n_iter_ : int + Actual number of iterations used in the solver. + """ + def __init__(self, *, alpha=1.0, fit_intercept=True, max_iter=100, + tol=1e-4, warm_start=False, verbose=0): + + super().__init__(alpha=alpha, fit_intercept=fit_intercept, + family="poisson", link='log', max_iter=max_iter, + tol=tol, warm_start=warm_start, verbose=verbose) + + @property + def family(self): + # Make this attribute read-only to avoid mis-uses e.g. in GridSearch. + return "poisson" + + @family.setter + def family(self, value): + if value != "poisson": + raise ValueError("PoissonRegressor.family must be 'poisson'!") + + +class GammaRegressor(GeneralizedLinearRegressor): + """Generalized Linear Model with a Gamma distribution. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + alpha : float, default=1 + Constant that multiplies the penalty term and thus determines the + regularization strength. ``alpha = 0`` is equivalent to unpenalized + GLMs. In this case, the design matrix `X` must have full column rank + (no collinearities). + + fit_intercept : bool, default=True + Specifies if a constant (a.k.a. bias or intercept) should be + added to the linear predictor (X @ coef + intercept). + + max_iter : int, default=100 + The maximal number of iterations for the solver. + + tol : float, default=1e-4 + Stopping criterion. For the lbfgs solver, + the iteration will stop when ``max{|g_j|, j = 1, ..., d} <= tol`` + where ``g_j`` is the j-th component of the gradient (derivative) of + the objective function. + + warm_start : bool, default=False + If set to ``True``, reuse the solution of the previous call to ``fit`` + as initialization for ``coef_`` and ``intercept_`` . + + verbose : int, default=0 + For the lbfgs solver set verbose to any positive number for verbosity. + + Attributes + ---------- + coef_ : array of shape (n_features,) + Estimated coefficients for the linear predictor (`X * coef_ + + intercept_`) in the GLM. + + intercept_ : float + Intercept (a.k.a. bias) added to linear predictor. + + n_iter_ : int + Actual number of iterations used in the solver. + """ + def __init__(self, *, alpha=1.0, fit_intercept=True, max_iter=100, + tol=1e-4, warm_start=False, verbose=0): + + super().__init__(alpha=alpha, fit_intercept=fit_intercept, + family="gamma", link='log', max_iter=max_iter, + tol=tol, warm_start=warm_start, verbose=verbose) + + @property + def family(self): + # Make this attribute read-only to avoid mis-uses e.g. in GridSearch. + return "gamma" + + @family.setter + def family(self, value): + if value != "gamma": + raise ValueError("GammaRegressor.family must be 'gamma'!") + + +class TweedieRegressor(GeneralizedLinearRegressor): + """Generalized Linear Model with a Tweedie distribution. + + This estimator can be used to model different GLMs depending on the + ``power`` parameter, which determines the underlying distribution. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + power : float, default=0 + The power determines the underlying target distribution according + to the following table: + + +-------+------------------------+ + | Power | Distribution | + +=======+========================+ + | 0 | Normal | + +-------+------------------------+ + | 1 | Poisson | + +-------+------------------------+ + | (1,2) | Compound Poisson Gamma | + +-------+------------------------+ + | 2 | Gamma | + +-------+------------------------+ + | 3 | Inverse Gaussian | + +-------+------------------------+ + + For ``0 < power < 1``, no distribution exists. + + alpha : float, default=1 + Constant that multiplies the penalty term and thus determines the + regularization strength. ``alpha = 0`` is equivalent to unpenalized + GLMs. In this case, the design matrix `X` must have full column rank + (no collinearities). + + link : {'auto', 'identity', 'log'}, default='auto' + The link function of the GLM, i.e. mapping from linear predictor + `X @ coeff + intercept` to prediction `y_pred`. Option 'auto' sets + the link depending on the chosen family as follows: + + - 'identity' for Normal distribution + - 'log' for Poisson, Gamma and Inverse Gaussian distributions + + fit_intercept : bool, default=True + Specifies if a constant (a.k.a. bias or intercept) should be + added to the linear predictor (X @ coef + intercept). + + max_iter : int, default=100 + The maximal number of iterations for the solver. + + tol : float, default=1e-4 + Stopping criterion. For the lbfgs solver, + the iteration will stop when ``max{|g_j|, j = 1, ..., d} <= tol`` + where ``g_j`` is the j-th component of the gradient (derivative) of + the objective function. + + warm_start : bool, default=False + If set to ``True``, reuse the solution of the previous call to ``fit`` + as initialization for ``coef_`` and ``intercept_`` . + + verbose : int, default=0 + For the lbfgs solver set verbose to any positive number for verbosity. + + Attributes + ---------- + coef_ : array of shape (n_features,) + Estimated coefficients for the linear predictor (`X @ coef_ + + intercept_`) in the GLM. + + intercept_ : float + Intercept (a.k.a. bias) added to linear predictor. + + n_iter_ : int + Actual number of iterations used in the solver. + """ + def __init__(self, *, power=0.0, alpha=1.0, fit_intercept=True, + link='auto', max_iter=100, tol=1e-4, + warm_start=False, verbose=0): + + super().__init__(alpha=alpha, fit_intercept=fit_intercept, + family=TweedieDistribution(power=power), link=link, + max_iter=max_iter, tol=tol, + warm_start=warm_start, verbose=verbose) + + @property + def family(self): + # We use a property with a setter to make sure that the family is + # always a Tweedie distribution, and that self.power and + # self.family.power are identical by construction. + dist = TweedieDistribution(power=self.power) + # TODO: make the returned object immutable + return dist + + @family.setter + def family(self, value): + if isinstance(value, TweedieDistribution): + self.power = value.power + else: + raise TypeError("TweedieRegressor.family must be of type " + "TweedieDistribution!") diff --git a/sklearn/linear_model/_glm/link.py b/sklearn/linear_model/_glm/link.py new file mode 100644 index 0000000000000..878d8e835bc42 --- /dev/null +++ b/sklearn/linear_model/_glm/link.py @@ -0,0 +1,110 @@ +""" +Link functions used in GLM +""" + +# Author: Christian Lorentzen +# License: BSD 3 clause + +from abc import ABCMeta, abstractmethod + +import numpy as np +from scipy.special import expit, logit + + +class BaseLink(metaclass=ABCMeta): + """Abstract base class for Link functions.""" + + @abstractmethod + def __call__(self, y_pred): + """Compute the link function g(y_pred). + + The link function links the mean y_pred=E[Y] to the so called linear + predictor (X*w), i.e. g(y_pred) = linear predictor. + + Parameters + ---------- + y_pred : array of shape (n_samples,) + Usually the (predicted) mean. + """ + + @abstractmethod + def derivative(self, y_pred): + """Compute the derivative of the link g'(y_pred). + + Parameters + ---------- + y_pred : array of shape (n_samples,) + Usually the (predicted) mean. + """ + + @abstractmethod + def inverse(self, lin_pred): + """Compute the inverse link function h(lin_pred). + + Gives the inverse relationship between linear predictor and the mean + y_pred=E[Y], i.e. h(linear predictor) = y_pred. + + Parameters + ---------- + lin_pred : array of shape (n_samples,) + Usually the (fitted) linear predictor. + """ + + @abstractmethod + def inverse_derivative(self, lin_pred): + """Compute the derivative of the inverse link function h'(lin_pred). + + Parameters + ---------- + lin_pred : array of shape (n_samples,) + Usually the (fitted) linear predictor. + """ + + +class IdentityLink(BaseLink): + """The identity link function g(x)=x.""" + + def __call__(self, y_pred): + return y_pred + + def derivative(self, y_pred): + return np.ones_like(y_pred) + + def inverse(self, lin_pred): + return lin_pred + + def inverse_derivative(self, lin_pred): + return np.ones_like(lin_pred) + + +class LogLink(BaseLink): + """The log link function g(x)=log(x).""" + + def __call__(self, y_pred): + return np.log(y_pred) + + def derivative(self, y_pred): + return 1 / y_pred + + def inverse(self, lin_pred): + return np.exp(lin_pred) + + def inverse_derivative(self, lin_pred): + return np.exp(lin_pred) + + +class LogitLink(BaseLink): + """The logit link function g(x)=logit(x).""" + + def __call__(self, y_pred): + return logit(y_pred) + + def derivative(self, y_pred): + return 1 / (y_pred * (1 - y_pred)) + + def inverse(self, lin_pred): + return expit(lin_pred) + + def inverse_derivative(self, lin_pred): + ep = expit(lin_pred) + return ep * (1 - ep) diff --git a/sklearn/linear_model/_glm/tests/__init__.py b/sklearn/linear_model/_glm/tests/__init__.py new file mode 100644 index 0000000000000..588cf7e93eef0 --- /dev/null +++ b/sklearn/linear_model/_glm/tests/__init__.py @@ -0,0 +1 @@ +# License: BSD 3 clause diff --git a/sklearn/linear_model/_glm/tests/test_glm.py b/sklearn/linear_model/_glm/tests/test_glm.py new file mode 100644 index 0000000000000..ece8f09c76acd --- /dev/null +++ b/sklearn/linear_model/_glm/tests/test_glm.py @@ -0,0 +1,431 @@ +# Authors: Christian Lorentzen +# +# License: BSD 3 clause + +import numpy as np +from numpy.testing import assert_allclose +import pytest +import warnings + +from sklearn.datasets import make_regression +from sklearn.linear_model._glm import GeneralizedLinearRegressor +from sklearn.linear_model import ( + TweedieRegressor, + PoissonRegressor, + GammaRegressor +) +from sklearn.linear_model._glm.link import ( + IdentityLink, + LogLink, +) +from sklearn._loss.glm_distribution import ( + TweedieDistribution, + NormalDistribution, PoissonDistribution, + GammaDistribution, InverseGaussianDistribution, +) +from sklearn.linear_model import Ridge +from sklearn.exceptions import ConvergenceWarning +from sklearn.model_selection import train_test_split + + +@pytest.fixture(scope="module") +def regression_data(): + X, y = make_regression(n_samples=107, + n_features=10, + n_informative=80, noise=0.5, + random_state=2) + return X, y + + +def test_sample_weights_validation(): + """Test the raised errors in the validation of sample_weight.""" + # scalar value but not positive + X = [[1]] + y = [1] + weights = 0 + glm = GeneralizedLinearRegressor() + + # Positive weights are accepted + glm.fit(X, y, sample_weight=1) + + # 2d array + weights = [[0]] + with pytest.raises(ValueError, match="must be 1D array or scalar"): + glm.fit(X, y, weights) + + # 1d but wrong length + weights = [1, 0] + msg = r"sample_weight.shape == \(2,\), expected \(1,\)!" + with pytest.raises(ValueError, match=msg): + glm.fit(X, y, weights) + + +@pytest.mark.parametrize('name, instance', + [('normal', NormalDistribution()), + ('poisson', PoissonDistribution()), + ('gamma', GammaDistribution()), + ('inverse-gaussian', InverseGaussianDistribution())]) +def test_glm_family_argument(name, instance): + """Test GLM family argument set as string.""" + y = np.array([0.1, 0.5]) # in range of all distributions + X = np.array([[1], [2]]) + glm = GeneralizedLinearRegressor(family=name, alpha=0).fit(X, y) + assert isinstance(glm._family_instance, instance.__class__) + + glm = GeneralizedLinearRegressor(family='not a family') + with pytest.raises(ValueError, match="family must be"): + glm.fit(X, y) + + +@pytest.mark.parametrize('name, instance', + [('identity', IdentityLink()), + ('log', LogLink())]) +def test_glm_link_argument(name, instance): + """Test GLM link argument set as string.""" + y = np.array([0.1, 0.5]) # in range of all distributions + X = np.array([[1], [2]]) + glm = GeneralizedLinearRegressor(family='normal', link=name).fit(X, y) + assert isinstance(glm._link_instance, instance.__class__) + + glm = GeneralizedLinearRegressor(family='normal', link='not a link') + with pytest.raises(ValueError, match="link must be"): + glm.fit(X, y) + + +@pytest.mark.parametrize('family, expected_link_class', [ + ('normal', IdentityLink), + ('poisson', LogLink), + ('gamma', LogLink), + ('inverse-gaussian', LogLink), +]) +def test_glm_link_auto(family, expected_link_class): + # Make sure link='auto' delivers the expected link function + y = np.array([0.1, 0.5]) # in range of all distributions + X = np.array([[1], [2]]) + glm = GeneralizedLinearRegressor(family=family, link='auto').fit(X, y) + assert isinstance(glm._link_instance, expected_link_class) + + +@pytest.mark.parametrize('alpha', ['not a number', -4.2]) +def test_glm_alpha_argument(alpha): + """Test GLM for invalid alpha argument.""" + y = np.array([1, 2]) + X = np.array([[1], [2]]) + glm = GeneralizedLinearRegressor(family='normal', alpha=alpha) + with pytest.raises(ValueError, + match="Penalty term must be a non-negative"): + glm.fit(X, y) + + +@pytest.mark.parametrize('fit_intercept', ['not bool', 1, 0, [True]]) +def test_glm_fit_intercept_argument(fit_intercept): + """Test GLM for invalid fit_intercept argument.""" + y = np.array([1, 2]) + X = np.array([[1], [1]]) + glm = GeneralizedLinearRegressor(fit_intercept=fit_intercept) + with pytest.raises(ValueError, match="fit_intercept must be bool"): + glm.fit(X, y) + + +@pytest.mark.parametrize('solver', + ['not a solver', 1, [1]]) +def test_glm_solver_argument(solver): + """Test GLM for invalid solver argument.""" + y = np.array([1, 2]) + X = np.array([[1], [2]]) + glm = GeneralizedLinearRegressor(solver=solver) + with pytest.raises(ValueError): + glm.fit(X, y) + + +@pytest.mark.parametrize('max_iter', ['not a number', 0, -1, 5.5, [1]]) +def test_glm_max_iter_argument(max_iter): + """Test GLM for invalid max_iter argument.""" + y = np.array([1, 2]) + X = np.array([[1], [2]]) + glm = GeneralizedLinearRegressor(max_iter=max_iter) + with pytest.raises(ValueError, match="must be a positive integer"): + glm.fit(X, y) + + +@pytest.mark.parametrize('tol', ['not a number', 0, -1.0, [1e-3]]) +def test_glm_tol_argument(tol): + """Test GLM for invalid tol argument.""" + y = np.array([1, 2]) + X = np.array([[1], [2]]) + glm = GeneralizedLinearRegressor(tol=tol) + with pytest.raises(ValueError, match="stopping criteria must be positive"): + glm.fit(X, y) + + +@pytest.mark.parametrize('warm_start', ['not bool', 1, 0, [True]]) +def test_glm_warm_start_argument(warm_start): + """Test GLM for invalid warm_start argument.""" + y = np.array([1, 2]) + X = np.array([[1], [1]]) + glm = GeneralizedLinearRegressor(warm_start=warm_start) + with pytest.raises(ValueError, match="warm_start must be bool"): + glm.fit(X, y) + + +@pytest.mark.parametrize('fit_intercept', [False, True]) +def test_glm_identity_regression(fit_intercept): + """Test GLM regression with identity link on a simple dataset.""" + coef = [1., 2.] + X = np.array([[1, 1, 1, 1, 1], [0, 1, 2, 3, 4]]).T + y = np.dot(X, coef) + glm = GeneralizedLinearRegressor(alpha=0, family='normal', link='identity', + fit_intercept=fit_intercept, tol=1e-12) + if fit_intercept: + glm.fit(X[:, 1:], y) + assert_allclose(glm.coef_, coef[1:], rtol=1e-10) + assert_allclose(glm.intercept_, coef[0], rtol=1e-10) + else: + glm.fit(X, y) + assert_allclose(glm.coef_, coef, rtol=1e-12) + + +@pytest.mark.parametrize('fit_intercept', [False, True]) +@pytest.mark.parametrize('alpha', [0.0, 1.0]) +@pytest.mark.parametrize('family', ['normal', 'poisson', 'gamma']) +def test_glm_sample_weight_consistentcy(fit_intercept, alpha, family): + """Test that the impact of sample_weight is consistent""" + rng = np.random.RandomState(0) + n_samples, n_features = 10, 5 + + X = rng.rand(n_samples, n_features) + y = rng.rand(n_samples) + glm_params = dict(alpha=alpha, family=family, link='auto', + fit_intercept=fit_intercept) + + glm = GeneralizedLinearRegressor(**glm_params).fit(X, y) + coef = glm.coef_.copy() + + # sample_weight=np.ones(..) should be equivalent to sample_weight=None + sample_weight = np.ones(y.shape) + glm.fit(X, y, sample_weight=sample_weight) + assert_allclose(glm.coef_, coef, rtol=1e-12) + + # sample_weight are normalized to 1 so, scaling them has no effect + sample_weight = 2*np.ones(y.shape) + glm.fit(X, y, sample_weight=sample_weight) + assert_allclose(glm.coef_, coef, rtol=1e-12) + + # setting one element of sample_weight to 0 is equivalent to removing + # the correspoding sample + sample_weight = np.ones(y.shape) + sample_weight[-1] = 0 + glm.fit(X, y, sample_weight=sample_weight) + coef1 = glm.coef_.copy() + glm.fit(X[:-1], y[:-1]) + assert_allclose(glm.coef_, coef1, rtol=1e-12) + + # check that multiplying sample_weight by 2 is equivalent + # to repeating correspoding samples twice + X2 = np.concatenate([X, X[:n_samples//2]], axis=0) + y2 = np.concatenate([y, y[:n_samples//2]]) + sample_weight_1 = np.ones(len(y)) + sample_weight_1[:n_samples//2] = 2 + + glm1 = GeneralizedLinearRegressor(**glm_params).fit( + X, y, sample_weight=sample_weight_1 + ) + + glm2 = GeneralizedLinearRegressor(**glm_params).fit( + X2, y2, sample_weight=None + ) + assert_allclose(glm1.coef_, glm2.coef_) + + +@pytest.mark.parametrize('fit_intercept', [True, False]) +@pytest.mark.parametrize( + 'family', + [NormalDistribution(), PoissonDistribution(), + GammaDistribution(), InverseGaussianDistribution(), + TweedieDistribution(power=1.5), TweedieDistribution(power=4.5)]) +def test_glm_log_regression(fit_intercept, family): + """Test GLM regression with log link on a simple dataset.""" + coef = [0.2, -0.1] + X = np.array([[1, 1, 1, 1, 1], [0, 1, 2, 3, 4]]).T + y = np.exp(np.dot(X, coef)) + glm = GeneralizedLinearRegressor( + alpha=0, family=family, link='log', + fit_intercept=fit_intercept, tol=1e-7) + if fit_intercept: + res = glm.fit(X[:, 1:], y) + assert_allclose(res.coef_, coef[1:], rtol=1e-6) + assert_allclose(res.intercept_, coef[0], rtol=1e-6) + else: + res = glm.fit(X, y) + assert_allclose(res.coef_, coef, rtol=2e-6) + + +@pytest.mark.parametrize('fit_intercept', [True, False]) +def test_warm_start(fit_intercept): + n_samples, n_features = 110, 10 + X, y = make_regression(n_samples=n_samples, n_features=n_features, + n_informative=n_features-2, noise=0.5, + random_state=42) + + glm1 = GeneralizedLinearRegressor( + warm_start=False, + fit_intercept=fit_intercept, + max_iter=1000 + ) + glm1.fit(X, y) + + glm2 = GeneralizedLinearRegressor( + warm_start=True, + fit_intercept=fit_intercept, + max_iter=1 + ) + # As we intentionally set max_iter=1, L-BFGS-B will issue a + # ConvergenceWarning which we here simply ignore. + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=ConvergenceWarning) + glm2.fit(X, y) + assert glm1.score(X, y) > glm2.score(X, y) + glm2.set_params(max_iter=1000) + glm2.fit(X, y) + # The two model are not exactly identical since the lbfgs solver + # computes the approximate hessian from previous iterations, which + # will not be strictly identical in the case of a warm start. + assert_allclose(glm1.coef_, glm2.coef_, rtol=1e-5) + assert_allclose(glm1.score(X, y), glm2.score(X, y), rtol=1e-4) + + +@pytest.mark.parametrize('n_samples, n_features', [(100, 10), (10, 100)]) +@pytest.mark.parametrize('fit_intercept', [True, False]) +@pytest.mark.parametrize('sample_weight', [None, True]) +def test_normal_ridge_comparison(n_samples, n_features, fit_intercept, + sample_weight, request): + """Compare with Ridge regression for Normal distributions.""" + test_size = 10 + X, y = make_regression(n_samples=n_samples + test_size, + n_features=n_features, + n_informative=n_features-2, noise=0.5, + random_state=42) + + if n_samples > n_features: + ridge_params = {"solver": "svd"} + else: + ridge_params = {"solver": "saga", "max_iter": 1000000, "tol": 1e-7} + + X_train, X_test, y_train, y_test, = train_test_split( + X, y, test_size=test_size, random_state=0 + ) + + alpha = 1.0 + if sample_weight is None: + sw_train = None + alpha_ridge = alpha * n_samples + else: + sw_train = np.random.RandomState(0).rand(len(y_train)) + alpha_ridge = alpha * sw_train.sum() + + # GLM has 1/(2*n) * Loss + 1/2*L2, Ridge has Loss + L2 + ridge = Ridge(alpha=alpha_ridge, normalize=False, + random_state=42, fit_intercept=fit_intercept, + **ridge_params) + ridge.fit(X_train, y_train, sample_weight=sw_train) + + glm = GeneralizedLinearRegressor(alpha=alpha, family='normal', + link='identity', + fit_intercept=fit_intercept, + max_iter=300, + tol=1e-5) + glm.fit(X_train, y_train, sample_weight=sw_train) + assert glm.coef_.shape == (X.shape[1], ) + assert_allclose(glm.coef_, ridge.coef_, atol=5e-5) + assert_allclose(glm.intercept_, ridge.intercept_, rtol=1e-5) + assert_allclose(glm.predict(X_train), ridge.predict(X_train), rtol=2e-4) + assert_allclose(glm.predict(X_test), ridge.predict(X_test), rtol=2e-4) + + +def test_poisson_glmnet(): + """Compare Poisson regression with L2 regularization and LogLink to glmnet + """ + # library("glmnet") + # options(digits=10) + # df <- data.frame(a=c(-2,-1,1,2), b=c(0,0,1,1), y=c(0,1,1,2)) + # x <- data.matrix(df[,c("a", "b")]) + # y <- df$y + # fit <- glmnet(x=x, y=y, alpha=0, intercept=T, family="poisson", + # standardize=F, thresh=1e-10, nlambda=10000) + # coef(fit, s=1) + # (Intercept) -0.12889386979 + # a 0.29019207995 + # b 0.03741173122 + X = np.array([[-2, -1, 1, 2], [0, 0, 1, 1]]).T + y = np.array([0, 1, 1, 2]) + glm = GeneralizedLinearRegressor(alpha=1, + fit_intercept=True, family='poisson', + link='log', tol=1e-7, + max_iter=300) + glm.fit(X, y) + assert_allclose(glm.intercept_, -0.12889386979, rtol=1e-5) + assert_allclose(glm.coef_, [0.29019207995, 0.03741173122], rtol=1e-5) + + +def test_convergence_warning(regression_data): + X, y = regression_data + + est = GeneralizedLinearRegressor(max_iter=1, tol=1e-20) + with pytest.warns(ConvergenceWarning): + est.fit(X, y) + + +def test_poisson_regression_family(regression_data): + # Make sure the family attribute is read-only to prevent searching over it + # e.g. in a grid search + est = PoissonRegressor() + est.family == "poisson" + + msg = "PoissonRegressor.family must be 'poisson'!" + with pytest.raises(ValueError, match=msg): + est.family = 0 + + +def test_gamma_regression_family(regression_data): + # Make sure the family attribute is read-only to prevent searching over it + # e.g. in a grid search + est = GammaRegressor() + est.family == "gamma" + + msg = "GammaRegressor.family must be 'gamma'!" + with pytest.raises(ValueError, match=msg): + est.family = 0 + + +def test_tweedie_regression_family(regression_data): + # Make sure the family attribute is always a TweedieDistribution and that + # the power attribute is properly updated + power = 2.0 + est = TweedieRegressor(power=power) + assert isinstance(est.family, TweedieDistribution) + assert est.family.power == power + assert est.power == power + + new_power = 0 + new_family = TweedieDistribution(power=new_power) + est.family = new_family + assert isinstance(est.family, TweedieDistribution) + assert est.family.power == new_power + assert est.power == new_power + + msg = "TweedieRegressor.family must be of type TweedieDistribution!" + with pytest.raises(TypeError, match=msg): + est.family = None + + +@pytest.mark.parametrize( + 'estimator, value', + [ + (PoissonRegressor(), True), + (GammaRegressor(), True), + (TweedieRegressor(power=1.5), True), + (TweedieRegressor(power=0), False) + ], +) +def test_tags(estimator, value): + assert estimator._get_tags()['requires_positive_y'] is value diff --git a/sklearn/linear_model/_glm/tests/test_link.py b/sklearn/linear_model/_glm/tests/test_link.py new file mode 100644 index 0000000000000..27ec4ed19bdc2 --- /dev/null +++ b/sklearn/linear_model/_glm/tests/test_link.py @@ -0,0 +1,45 @@ +# Authors: Christian Lorentzen +# +# License: BSD 3 clause +import numpy as np +from numpy.testing import assert_allclose +import pytest +from scipy.optimize import check_grad + +from sklearn.linear_model._glm.link import ( + IdentityLink, + LogLink, + LogitLink, +) + + +LINK_FUNCTIONS = [IdentityLink, LogLink, LogitLink] + + +@pytest.mark.parametrize('Link', LINK_FUNCTIONS) +def test_link_properties(Link): + """Test link inverse and derivative.""" + rng = np.random.RandomState(42) + x = rng.rand(100) * 100 + link = Link() + if isinstance(link, LogitLink): + # careful for large x, note expit(36) = 1 + # limit max eta to 15 + x = x / 100 * 15 + assert_allclose(link(link.inverse(x)), x) + # if g(h(x)) = x, then g'(h(x)) = 1/h'(x) + # g = link, h = link.inverse + assert_allclose(link.derivative(link.inverse(x)), + 1 / link.inverse_derivative(x)) + + +@pytest.mark.parametrize('Link', LINK_FUNCTIONS) +def test_link_derivative(Link): + link = Link() + x = np.random.RandomState(0).rand(1) + err = check_grad(link, link.derivative, x) / link.derivative(x) + assert abs(err) < 1e-6 + + err = (check_grad(link.inverse, link.inverse_derivative, x) + / link.derivative(x)) + assert abs(err) < 1e-6 diff --git a/sklearn/linear_model/_huber.py b/sklearn/linear_model/_huber.py index 06d182f7fcbdb..1d3a3fcc73421 100644 --- a/sklearn/linear_model/_huber.py +++ b/sklearn/linear_model/_huber.py @@ -205,7 +205,7 @@ class HuberRegressor(LinearModel, RegressorMixin, BaseEstimator): >>> y[:4] = rng.uniform(10, 20, 4) >>> huber = HuberRegressor().fit(X, y) >>> huber.score(X, y) - -7.284608623514573 + -7.284... >>> huber.predict(X[:1,]) array([806.7200...]) >>> linear = LinearRegression().fit(X, y) @@ -252,7 +252,7 @@ def fit(self, X, y, sample_weight=None): ------- self : object """ - X, y = check_X_y( + X, y = self._validate_data( X, y, copy=False, accept_sparse=['csr'], y_numeric=True, dtype=[np.float64, np.float32]) diff --git a/sklearn/linear_model/_least_angle.py b/sklearn/linear_model/_least_angle.py index b0be830eb76c6..9f0f62471376a 100644 --- a/sklearn/linear_model/_least_angle.py +++ b/sklearn/linear_model/_least_angle.py @@ -47,12 +47,6 @@ def lars_path(X, y, Xy=None, Gram=None, max_iter=500, alpha_min=0, Input data. Note that if X is None then the Gram matrix must be specified, i.e., cannot be None or False. - .. deprecated:: 0.21 - - The use of ``X`` is ``None`` in combination with ``Gram`` is not - ``None`` will be removed in v0.23. Use :func:`lars_path_gram` - instead. - y : None or array-like of shape (n_samples,) Input targets. @@ -67,11 +61,6 @@ def lars_path(X, y, Xy=None, Gram=None, max_iter=500, alpha_min=0, matrix is precomputed from the given X, if there are more samples than features. - .. deprecated:: 0.21 - - The use of ``X`` is ``None`` in combination with ``Gram`` is not - None will be removed in v0.23. Use :func:`lars_path_gram` instead. - max_iter : int, default=500 Maximum number of iterations to perform, set to infinity for no limit. @@ -155,9 +144,10 @@ def lars_path(X, y, Xy=None, Gram=None, max_iter=500, alpha_min=0, """ if X is None and Gram is not None: - warnings.warn('Use lars_path_gram to avoid passing X and y. ' - 'The current option will be removed in v0.23.', - FutureWarning) + raise ValueError( + 'X cannot be None if Gram is not None' + 'Use lars_path_gram to avoid passing X and y.' + ) return _lars_path_solver( X=X, y=y, Xy=Xy, Gram=Gram, n_samples=None, max_iter=max_iter, alpha_min=alpha_min, method=method, copy_X=copy_X, @@ -954,7 +944,7 @@ def fit(self, X, y, Xy=None): self : object returns an instance of self. """ - X, y = check_X_y(X, y, y_numeric=True, multi_output=True) + X, y = self._validate_data(X, y, y_numeric=True, multi_output=True) alpha = getattr(self, 'alpha', 0.) if hasattr(self, 'n_nonzero_coefs'): @@ -1377,7 +1367,7 @@ def fit(self, X, y): self : object returns an instance of self. """ - X, y = check_X_y(X, y, y_numeric=True) + X, y = self._validate_data(X, y, y_numeric=True) X = as_float_array(X, copy=self.copy_X) y = as_float_array(y, copy=self.copy_X) @@ -1758,7 +1748,7 @@ def fit(self, X, y, copy_X=None): """ if copy_X is None: copy_X = self.copy_X - X, y = check_X_y(X, y, y_numeric=True) + X, y = self._validate_data(X, y, y_numeric=True) X, y, Xmean, ymean, Xstd = LinearModel._preprocess_data( X, y, self.fit_intercept, self.normalize, copy_X) diff --git a/sklearn/linear_model/_logistic.py b/sklearn/linear_model/_logistic.py index 7921150e0fa01..9e84e56ee0284 100644 --- a/sklearn/linear_model/_logistic.py +++ b/sklearn/linear_model/_logistic.py @@ -38,6 +38,12 @@ from ..metrics import get_scorer +_LOGISTIC_SOLVER_CONVERGENCE_MSG = ( + "Please also refer to the documentation for alternative solver options:\n" + " https://scikit-learn.org/stable/modules/linear_model.html" + "#logistic-regression") + + # .. some helper functions for logistic_regression_path .. def _intercept_dot(w, X, y): """Computes y * np.dot(X, w). @@ -46,18 +52,18 @@ def _intercept_dot(w, X, y): Parameters ---------- - w : ndarray, shape (n_features,) or (n_features + 1,) + w : ndarray of shape (n_features,) or (n_features + 1,) Coefficient vector. - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data. - y : ndarray, shape (n_samples,) + y : ndarray of shape (n_samples,) Array of labels. Returns ------- - w : ndarray, shape (n_features,) + w : ndarray of shape (n_features,) Coefficient vector without the intercept weight (w[-1]) if the intercept should be fit. Unchanged otherwise. @@ -82,19 +88,19 @@ def _logistic_loss_and_grad(w, X, y, alpha, sample_weight=None): Parameters ---------- - w : ndarray, shape (n_features,) or (n_features + 1,) + w : ndarray of shape (n_features,) or (n_features + 1,) Coefficient vector. - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data. - y : ndarray, shape (n_samples,) + y : ndarray of shape (n_samples,) Array of labels. alpha : float Regularization parameter. alpha is equal to 1 / C. - sample_weight : array-like, shape (n_samples,) optional + sample_weight : array-like of shape (n_samples,), default=None Array of weights that are assigned to individual samples. If not provided, then each sample is given unit weight. @@ -103,7 +109,7 @@ def _logistic_loss_and_grad(w, X, y, alpha, sample_weight=None): out : float Logistic loss. - grad : ndarray, shape (n_features,) or (n_features + 1,) + grad : ndarray of shape (n_features,) or (n_features + 1,) Logistic gradient. """ n_samples, n_features = X.shape @@ -133,19 +139,19 @@ def _logistic_loss(w, X, y, alpha, sample_weight=None): Parameters ---------- - w : ndarray, shape (n_features,) or (n_features + 1,) + w : ndarray of shape (n_features,) or (n_features + 1,) Coefficient vector. - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data. - y : ndarray, shape (n_samples,) + y : ndarray of shape (n_samples,) Array of labels. alpha : float Regularization parameter. alpha is equal to 1 / C. - sample_weight : array-like, shape (n_samples,) optional + sample_weight : array-like of shape (n_samples,) default=None Array of weights that are assigned to individual samples. If not provided, then each sample is given unit weight. @@ -169,25 +175,25 @@ def _logistic_grad_hess(w, X, y, alpha, sample_weight=None): Parameters ---------- - w : ndarray, shape (n_features,) or (n_features + 1,) + w : ndarray of shape (n_features,) or (n_features + 1,) Coefficient vector. - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data. - y : ndarray, shape (n_samples,) + y : ndarray of shape (n_samples,) Array of labels. alpha : float Regularization parameter. alpha is equal to 1 / C. - sample_weight : array-like, shape (n_samples,) optional + sample_weight : array-like of shape (n_samples,) default=None Array of weights that are assigned to individual samples. If not provided, then each sample is given unit weight. Returns ------- - grad : ndarray, shape (n_features,) or (n_features + 1,) + grad : ndarray of shape (n_features,) or (n_features + 1,) Logistic gradient. Hs : callable @@ -246,20 +252,20 @@ def _multinomial_loss(w, X, Y, alpha, sample_weight): Parameters ---------- - w : ndarray, shape (n_classes * n_features,) or + w : ndarray of shape (n_classes * n_features,) or (n_classes * (n_features + 1),) Coefficient vector. - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data. - Y : ndarray, shape (n_samples, n_classes) + Y : ndarray of shape (n_samples, n_classes) Transformed labels according to the output of LabelBinarizer. alpha : float Regularization parameter. alpha is equal to 1 / C. - sample_weight : array-like, shape (n_samples,) + sample_weight : array-like of shape (n_samples,) Array of weights that are assigned to individual samples. Returns @@ -267,10 +273,10 @@ def _multinomial_loss(w, X, Y, alpha, sample_weight): loss : float Multinomial loss. - p : ndarray, shape (n_samples, n_classes) + p : ndarray of shape (n_samples, n_classes) Estimated class probabilities. - w : ndarray, shape (n_classes, n_features) + w : ndarray of shape (n_classes, n_features) Reshaped param vector excluding intercept terms. Reference @@ -302,20 +308,20 @@ def _multinomial_loss_grad(w, X, Y, alpha, sample_weight): Parameters ---------- - w : ndarray, shape (n_classes * n_features,) or + w : ndarray of shape (n_classes * n_features,) or (n_classes * (n_features + 1),) Coefficient vector. - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data. - Y : ndarray, shape (n_samples, n_classes) + Y : ndarray of shape (n_samples, n_classes) Transformed labels according to the output of LabelBinarizer. alpha : float Regularization parameter. alpha is equal to 1 / C. - sample_weight : array-like, shape (n_samples,) + sample_weight : array-like of shape (n_samples,) Array of weights that are assigned to individual samples. Returns @@ -323,11 +329,11 @@ def _multinomial_loss_grad(w, X, Y, alpha, sample_weight): loss : float Multinomial loss. - grad : ndarray, shape (n_classes * n_features,) or - (n_classes * (n_features + 1),) + grad : ndarray of shape (n_classes * n_features,) or \ + (n_classes * (n_features + 1),) Ravelled gradient of the multinomial loss. - p : ndarray, shape (n_samples, n_classes) + p : ndarray of shape (n_samples, n_classes) Estimated class probabilities Reference @@ -356,26 +362,26 @@ def _multinomial_grad_hess(w, X, Y, alpha, sample_weight): Parameters ---------- - w : ndarray, shape (n_classes * n_features,) or + w : ndarray of shape (n_classes * n_features,) or (n_classes * (n_features + 1),) Coefficient vector. - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data. - Y : ndarray, shape (n_samples, n_classes) + Y : ndarray of shape (n_samples, n_classes) Transformed labels according to the output of LabelBinarizer. alpha : float Regularization parameter. alpha is equal to 1 / C. - sample_weight : array-like, shape (n_samples,) + sample_weight : array-like of shape (n_samples,) Array of weights that are assigned to individual samples. Returns ------- - grad : array, shape (n_classes * n_features,) or - (n_classes * (n_features + 1),) + grad : ndarray of shape (n_classes * n_features,) or \ + (n_classes * (n_features + 1),) Ravelled gradient of the multinomial loss. hessp : callable @@ -470,178 +476,6 @@ def _check_multi_class(multi_class, solver, n_classes): return multi_class -@deprecated('logistic_regression_path was deprecated in version 0.21 and ' - 'will be removed in version 0.23.0') -def logistic_regression_path(X, y, pos_class=None, Cs=10, fit_intercept=True, - max_iter=100, tol=1e-4, verbose=0, - solver='lbfgs', coef=None, - class_weight=None, dual=False, penalty='l2', - intercept_scaling=1., multi_class='auto', - random_state=None, check_input=True, - max_squared_sum=None, sample_weight=None, - l1_ratio=None): - """Compute a Logistic Regression model for a list of regularization - parameters. - - This is an implementation that uses the result of the previous model - to speed up computations along the set of solutions, making it faster - than sequentially calling LogisticRegression for the different parameters. - Note that there will be no speedup with liblinear solver, since it does - not handle warm-starting. - - .. deprecated:: 0.21 - ``logistic_regression_path`` was deprecated in version 0.21 and will - be removed in 0.23. - - Read more in the :ref:`User Guide `. - - Parameters - ---------- - X : array-like or sparse matrix, shape (n_samples, n_features) - Input data. - - y : array-like, shape (n_samples,) or (n_samples, n_targets) - Input data, target values. - - pos_class : int, None - The class with respect to which we perform a one-vs-all fit. - If None, then it is assumed that the given problem is binary. - - Cs : int | array-like, shape (n_cs,) - List of values for the regularization parameter or integer specifying - the number of regularization parameters that should be used. In this - case, the parameters will be chosen in a logarithmic scale between - 1e-4 and 1e4. - - fit_intercept : bool - Whether to fit an intercept for the model. In this case the shape of - the returned array is (n_cs, n_features + 1). - - max_iter : int - Maximum number of iterations for the solver. - - tol : float - Stopping criterion. For the newton-cg and lbfgs solvers, the iteration - will stop when ``max{|g_i | i = 1, ..., n} <= tol`` - where ``g_i`` is the i-th component of the gradient. - - verbose : int - For the liblinear and lbfgs solvers set verbose to any positive - number for verbosity. - - solver : {'lbfgs', 'newton-cg', 'liblinear', 'sag', 'saga'} - Numerical solver to use. - - coef : array-like, shape (n_features,), default None - Initialization value for coefficients of logistic regression. - Useless for liblinear solver. - - class_weight : dict or 'balanced', optional - Weights associated with classes in the form ``{class_label: weight}``. - If not given, all classes are supposed to have weight one. - - The "balanced" mode uses the values of y to automatically adjust - weights inversely proportional to class frequencies in the input data - as ``n_samples / (n_classes * np.bincount(y))``. - - Note that these weights will be multiplied with sample_weight (passed - through the fit method) if sample_weight is specified. - - dual : bool - Dual or primal formulation. Dual formulation is only implemented for - l2 penalty with liblinear solver. Prefer dual=False when - n_samples > n_features. - - penalty : str, 'l1', 'l2', or 'elasticnet' - Used to specify the norm used in the penalization. The 'newton-cg', - 'sag' and 'lbfgs' solvers support only l2 penalties. 'elasticnet' is - only supported by the 'saga' solver. - - intercept_scaling : float, default 1. - Useful only when the solver 'liblinear' is used - and self.fit_intercept is set to True. In this case, x becomes - [x, self.intercept_scaling], - i.e. a "synthetic" feature with constant value equal to - intercept_scaling is appended to the instance vector. - The intercept becomes ``intercept_scaling * synthetic_feature_weight``. - - Note! the synthetic feature weight is subject to l1/l2 regularization - as all other features. - To lessen the effect of regularization on synthetic feature weight - (and therefore on the intercept) intercept_scaling has to be increased. - - multi_class : {'ovr', 'multinomial', 'auto'}, default='auto' - If the option chosen is 'ovr', then a binary problem is fit for each - label. For 'multinomial' the loss minimised is the multinomial loss fit - across the entire probability distribution, *even when the data is - binary*. 'multinomial' is unavailable when solver='liblinear'. - 'auto' selects 'ovr' if the data is binary, or if solver='liblinear', - and otherwise selects 'multinomial'. - - .. versionadded:: 0.18 - Stochastic Average Gradient descent solver for 'multinomial' case. - .. versionchanged:: 0.22 - Default changed from 'ovr' to 'auto' in 0.22. - - random_state : int, RandomState instance or None, optional, default None - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. Used when ``solver`` == 'sag' or - 'liblinear'. - - check_input : bool, default True - If False, the input arrays X and y will not be checked. - - max_squared_sum : float, default None - Maximum squared sum of X over samples. Used only in SAG solver. - If None, it will be computed, going through all the samples. - The value should be precomputed to speed up cross validation. - - sample_weight : array-like, shape(n_samples,) optional - Array of weights that are assigned to individual samples. - If not provided, then each sample is given unit weight. - - l1_ratio : float or None, optional (default=None) - The Elastic-Net mixing parameter, with ``0 <= l1_ratio <= 1``. Only - used if ``penalty='elasticnet'``. Setting ``l1_ratio=0`` is equivalent - to using ``penalty='l2'``, while setting ``l1_ratio=1`` is equivalent - to using ``penalty='l1'``. For ``0 < l1_ratio <1``, the penalty is a - combination of L1 and L2. - - Returns - ------- - coefs : ndarray, shape (n_cs, n_features) or (n_cs, n_features + 1) - List of coefficients for the Logistic Regression model. If - fit_intercept is set to True then the second dimension will be - n_features + 1, where the last item represents the intercept. For - ``multiclass='multinomial'``, the shape is (n_classes, n_cs, - n_features) or (n_classes, n_cs, n_features + 1). - - Cs : ndarray - Grid of Cs used for cross-validation. - - n_iter : array, shape (n_cs,) - Actual number of iteration for each Cs. - - Notes - ----- - You might get slightly different results with the solver liblinear than - with the others since this uses LIBLINEAR which penalizes the intercept. - - .. versionchanged:: 0.19 - The "copy" parameter was removed. - """ - - return _logistic_regression_path( - X, y, pos_class=None, Cs=10, fit_intercept=True, max_iter=100, - tol=1e-4, verbose=0, solver='lbfgs', coef=None, class_weight=None, - dual=False, penalty='l2', intercept_scaling=1., multi_class='auto', - random_state=None, check_input=True, max_squared_sum=None, - sample_weight=None, l1_ratio=None) - - def _logistic_regression_path(X, y, pos_class=None, Cs=10, fit_intercept=True, max_iter=100, tol=1e-4, verbose=0, solver='lbfgs', coef=None, @@ -663,46 +497,47 @@ def _logistic_regression_path(X, y, pos_class=None, Cs=10, fit_intercept=True, Parameters ---------- - X : array-like or sparse matrix, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Input data. - y : array-like, shape (n_samples,) or (n_samples, n_targets) + y : array-like of shape (n_samples,) or (n_samples, n_targets) Input data, target values. - pos_class : int, None + pos_class : int, default=None The class with respect to which we perform a one-vs-all fit. If None, then it is assumed that the given problem is binary. - Cs : int | array-like, shape (n_cs,) + Cs : int or array-like of shape (n_cs,), default=10 List of values for the regularization parameter or integer specifying the number of regularization parameters that should be used. In this case, the parameters will be chosen in a logarithmic scale between 1e-4 and 1e4. - fit_intercept : bool + fit_intercept : bool, default=True Whether to fit an intercept for the model. In this case the shape of the returned array is (n_cs, n_features + 1). - max_iter : int + max_iter : int, default=100 Maximum number of iterations for the solver. - tol : float + tol : float, default=1e-4 Stopping criterion. For the newton-cg and lbfgs solvers, the iteration will stop when ``max{|g_i | i = 1, ..., n} <= tol`` where ``g_i`` is the i-th component of the gradient. - verbose : int + verbose : int, default=0 For the liblinear and lbfgs solvers set verbose to any positive number for verbosity. - solver : {'lbfgs', 'newton-cg', 'liblinear', 'sag', 'saga'} + solver : {'lbfgs', 'newton-cg', 'liblinear', 'sag', 'saga'}, \ + default='lbfgs' Numerical solver to use. - coef : array-like, shape (n_features,), default None + coef : array-like of shape (n_features,), default=None Initialization value for coefficients of logistic regression. Useless for liblinear solver. - class_weight : dict or 'balanced', optional + class_weight : dict or 'balanced', default=None Weights associated with classes in the form ``{class_label: weight}``. If not given, all classes are supposed to have weight one. @@ -713,17 +548,17 @@ def _logistic_regression_path(X, y, pos_class=None, Cs=10, fit_intercept=True, Note that these weights will be multiplied with sample_weight (passed through the fit method) if sample_weight is specified. - dual : bool + dual : bool, default=False Dual or primal formulation. Dual formulation is only implemented for l2 penalty with liblinear solver. Prefer dual=False when n_samples > n_features. - penalty : str, 'l1', 'l2', or 'elasticnet' + penalty : {'l1', 'l2', 'elasticnet'}, default='l2' Used to specify the norm used in the penalization. The 'newton-cg', 'sag' and 'lbfgs' solvers support only l2 penalties. 'elasticnet' is only supported by the 'saga' solver. - intercept_scaling : float, default 1. + intercept_scaling : float, default=1. Useful only when the solver 'liblinear' is used and self.fit_intercept is set to True. In this case, x becomes [x, self.intercept_scaling], @@ -749,27 +584,23 @@ def _logistic_regression_path(X, y, pos_class=None, Cs=10, fit_intercept=True, .. versionchanged:: 0.22 Default changed from 'ovr' to 'auto' in 0.22. - random_state : int, RandomState instance or None, optional, default None - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. Used when ``solver`` == 'sag' or - 'liblinear'. + random_state : int, RandomState instance, default=None + Used when ``solver`` == 'sag', 'saga' or 'liblinear' to shuffle the + data. See :term:`Glossary ` for details. - check_input : bool, default True + check_input : bool, default=True If False, the input arrays X and y will not be checked. - max_squared_sum : float, default None + max_squared_sum : float, default=None Maximum squared sum of X over samples. Used only in SAG solver. If None, it will be computed, going through all the samples. The value should be precomputed to speed up cross validation. - sample_weight : array-like, shape(n_samples,) optional + sample_weight : array-like of shape(n_samples,), default=None Array of weights that are assigned to individual samples. If not provided, then each sample is given unit weight. - l1_ratio : float or None, optional (default=None) + l1_ratio : float, default=None The Elastic-Net mixing parameter, with ``0 <= l1_ratio <= 1``. Only used if ``penalty='elasticnet'``. Setting ``l1_ratio=0`` is equivalent to using ``penalty='l2'``, while setting ``l1_ratio=1`` is equivalent @@ -778,7 +609,7 @@ def _logistic_regression_path(X, y, pos_class=None, Cs=10, fit_intercept=True, Returns ------- - coefs : ndarray, shape (n_cs, n_features) or (n_cs, n_features + 1) + coefs : ndarray of shape (n_cs, n_features) or (n_cs, n_features + 1) List of coefficients for the Logistic Regression model. If fit_intercept is set to True then the second dimension will be n_features + 1, where the last item represents the intercept. For @@ -788,7 +619,7 @@ def _logistic_regression_path(X, y, pos_class=None, Cs=10, fit_intercept=True, Cs : ndarray Grid of Cs used for cross-validation. - n_iter : array, shape (n_cs,) + n_iter : array of shape (n_cs,) Actual number of iteration for each Cs. Notes @@ -928,7 +759,9 @@ def _logistic_regression_path(X, y, pos_class=None, Cs=10, fit_intercept=True, args=(X, target, 1. / C, sample_weight), options={"iprint": iprint, "gtol": tol, "maxiter": max_iter} ) - n_iter_i = _check_optimize_result(solver, opt_res, max_iter) + n_iter_i = _check_optimize_result( + solver, opt_res, max_iter, + extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG) w0, loss = opt_res.x, opt_res.fun elif solver == 'newton-cg': args = (X, target, 1. / C, sample_weight) @@ -998,10 +831,10 @@ def _log_reg_scoring_path(X, y, train, test, pos_class=None, Cs=10, Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data. - y : array-like, shape (n_samples,) or (n_samples, n_targets) + y : array-like of shape (n_samples,) or (n_samples, n_targets) Target labels. train : list of indices @@ -1010,34 +843,34 @@ def _log_reg_scoring_path(X, y, train, test, pos_class=None, Cs=10, test : list of indices The indices of the test set. - pos_class : int, None + pos_class : int, default=None The class with respect to which we perform a one-vs-all fit. If None, then it is assumed that the given problem is binary. - Cs : list of floats | int + Cs : int or list of floats, default=10 Each of the values in Cs describes the inverse of regularization strength. If Cs is as an int, then a grid of Cs values are chosen in a logarithmic scale between 1e-4 and 1e4. If not provided, then a fixed set of values for Cs are used. - scoring : callable or None, optional, default: None + scoring : callable, default=None A string (see model evaluation documentation) or a scorer callable object / function with signature ``scorer(estimator, X, y)``. For a list of scoring functions that can be used, look at :mod:`sklearn.metrics`. The default scoring option used is accuracy_score. - fit_intercept : bool + fit_intercept : bool, default=False If False, then the bias term is set to zero. Else the last term of each coef_ gives us the intercept. - max_iter : int + max_iter : int, default=100 Maximum number of iterations for the solver. - tol : float + tol : float, default=1e-4 Tolerance for stopping criteria. - class_weight : dict or 'balanced', optional + class_weight : dict or 'balanced', default=None Weights associated with classes in the form ``{class_label: weight}``. If not given, all classes are supposed to have weight one. @@ -1048,24 +881,25 @@ def _log_reg_scoring_path(X, y, train, test, pos_class=None, Cs=10, Note that these weights will be multiplied with sample_weight (passed through the fit method) if sample_weight is specified. - verbose : int + verbose : int, default=0 For the liblinear and lbfgs solvers set verbose to any positive number for verbosity. - solver : {'lbfgs', 'newton-cg', 'liblinear', 'sag', 'saga'} + solver : {'lbfgs', 'newton-cg', 'liblinear', 'sag', 'saga'}, \ + default='lbfgs' Decides which solver to use. - penalty : str, 'l1', 'l2', or 'elasticnet' + penalty : {'l1', 'l2', 'elasticnet'}, default='l2' Used to specify the norm used in the penalization. The 'newton-cg', 'sag' and 'lbfgs' solvers support only l2 penalties. 'elasticnet' is only supported by the 'saga' solver. - dual : bool + dual : bool, default=False Dual or primal formulation. Dual formulation is only implemented for l2 penalty with liblinear solver. Prefer dual=False when n_samples > n_features. - intercept_scaling : float, default 1. + intercept_scaling : float, default=1. Useful only when the solver 'liblinear' is used and self.fit_intercept is set to True. In this case, x becomes [x, self.intercept_scaling], @@ -1077,30 +911,26 @@ def _log_reg_scoring_path(X, y, train, test, pos_class=None, Cs=10, To lessen the effect of regularization on synthetic feature weight (and therefore on the intercept) intercept_scaling has to be increased. - multi_class : {'ovr', 'multinomial'} + multi_class : {'auto', 'ovr', 'multinomial'}, default='auto' If the option chosen is 'ovr', then a binary problem is fit for each label. For 'multinomial' the loss minimised is the multinomial loss fit across the entire probability distribution, *even when the data is binary*. 'multinomial' is unavailable when solver='liblinear'. - random_state : int, RandomState instance or None, optional, default None - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. Used when ``solver`` == 'sag' and - 'liblinear'. + random_state : int, RandomState instance, default=None + Used when ``solver`` == 'sag', 'saga' or 'liblinear' to shuffle the + data. See :term:`Glossary ` for details. - max_squared_sum : float, default None + max_squared_sum : float, default=None Maximum squared sum of X over samples. Used only in SAG solver. If None, it will be computed, going through all the samples. The value should be precomputed to speed up cross validation. - sample_weight : array-like, shape(n_samples,) optional + sample_weight : array-like of shape(n_samples,), default=None Array of weights that are assigned to individual samples. If not provided, then each sample is given unit weight. - l1_ratio : float or None, optional (default=None) + l1_ratio : float, default=None The Elastic-Net mixing parameter, with ``0 <= l1_ratio <= 1``. Only used if ``penalty='elasticnet'``. Setting ``l1_ratio=0`` is equivalent to using ``penalty='l2'``, while setting ``l1_ratio=1`` is equivalent @@ -1109,7 +939,7 @@ def _log_reg_scoring_path(X, y, train, test, pos_class=None, Cs=10, Returns ------- - coefs : ndarray, shape (n_cs, n_features) or (n_cs, n_features + 1) + coefs : ndarray of shape (n_cs, n_features) or (n_cs, n_features + 1) List of coefficients for the Logistic Regression model. If fit_intercept is set to True then the second dimension will be n_features + 1, where the last item represents the intercept. @@ -1117,10 +947,10 @@ def _log_reg_scoring_path(X, y, train, test, pos_class=None, Cs=10, Cs : ndarray Grid of Cs used for cross-validation. - scores : ndarray, shape (n_cs,) + scores : ndarray of shape (n_cs,) Scores obtained for each Cs. - n_iter : array, shape(n_cs,) + n_iter : ndarray of shape(n_cs,) Actual number of iteration for each Cs. """ X_train = X[train] @@ -1206,7 +1036,7 @@ class LogisticRegression(BaseEstimator, LinearClassifierMixin, Parameters ---------- - penalty : str, 'l1', 'l2', 'elasticnet' or 'none', optional (default='l2') + penalty : {'l1', 'l2', 'elasticnet', 'none'}, default='l2' Used to specify the norm used in the penalization. The 'newton-cg', 'sag' and 'lbfgs' solvers support only l2 penalties. 'elasticnet' is only supported by the 'saga' solver. If 'none' (not supported by the @@ -1215,24 +1045,24 @@ class LogisticRegression(BaseEstimator, LinearClassifierMixin, .. versionadded:: 0.19 l1 penalty with SAGA solver (allowing 'multinomial' + L1) - dual : bool, optional (default=False) + dual : bool, default=False Dual or primal formulation. Dual formulation is only implemented for l2 penalty with liblinear solver. Prefer dual=False when n_samples > n_features. - tol : float, optional (default=1e-4) + tol : float, default=1e-4 Tolerance for stopping criteria. - C : float, optional (default=1.0) + C : float, default=1.0 Inverse of regularization strength; must be a positive float. Like in support vector machines, smaller values specify stronger regularization. - fit_intercept : bool, optional (default=True) + fit_intercept : bool, default=True Specifies if a constant (a.k.a. bias or intercept) should be added to the decision function. - intercept_scaling : float, optional (default=1) + intercept_scaling : float, default=1 Useful only when the solver 'liblinear' is used and self.fit_intercept is set to True. In this case, x becomes [x, self.intercept_scaling], @@ -1245,7 +1075,7 @@ class LogisticRegression(BaseEstimator, LinearClassifierMixin, To lessen the effect of regularization on synthetic feature weight (and therefore on the intercept) intercept_scaling has to be increased. - class_weight : dict or 'balanced', optional (default=None) + class_weight : dict or 'balanced', default=None Weights associated with classes in the form ``{class_label: weight}``. If not given, all classes are supposed to have weight one. @@ -1259,16 +1089,12 @@ class LogisticRegression(BaseEstimator, LinearClassifierMixin, .. versionadded:: 0.17 *class_weight='balanced'* - random_state : int, RandomState instance or None, optional (default=None) - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. Used when ``solver`` == 'sag' or - 'liblinear'. + random_state : int, RandomState instance, default=None + Used when ``solver`` == 'sag', 'saga' or 'liblinear' to shuffle the + data. See :term:`Glossary ` for details. - solver : str, {'newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'}, \ - optional (default='lbfgs') + solver : {'newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'}, \ + default='lbfgs' Algorithm to use in the optimization problem. @@ -1293,10 +1119,10 @@ class LogisticRegression(BaseEstimator, LinearClassifierMixin, .. versionchanged:: 0.22 The default solver changed from 'liblinear' to 'lbfgs' in 0.22. - max_iter : int, optional (default=100) + max_iter : int, default=100 Maximum number of iterations taken for the solvers to converge. - multi_class : {'ovr', 'multinomial', 'auto'}, default='auto' + multi_class : {'auto', 'ovr', 'multinomial'}, default='auto' If the option chosen is 'ovr', then a binary problem is fit for each label. For 'multinomial' the loss minimised is the multinomial loss fit across the entire probability distribution, *even when the data is @@ -1309,11 +1135,11 @@ class LogisticRegression(BaseEstimator, LinearClassifierMixin, .. versionchanged:: 0.22 Default changed from 'ovr' to 'auto' in 0.22. - verbose : int, optional (default=0) + verbose : int, default=0 For the liblinear and lbfgs solvers set verbose to any positive number for verbosity. - warm_start : bool, optional (default=False) + warm_start : bool, default=False When set to True, reuse the solution of the previous call to fit as initialization, otherwise, just erase the previous solution. Useless for liblinear solver. See :term:`the Glossary `. @@ -1321,7 +1147,7 @@ class LogisticRegression(BaseEstimator, LinearClassifierMixin, .. versionadded:: 0.17 *warm_start* to support *lbfgs*, *newton-cg*, *sag*, *saga* solvers. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None Number of CPU cores used when parallelizing over classes if multi_class='ovr'". This parameter is ignored when the ``solver`` is set to 'liblinear' regardless of whether 'multi_class' is specified or @@ -1329,9 +1155,9 @@ class LogisticRegression(BaseEstimator, LinearClassifierMixin, context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - l1_ratio : float or None, optional (default=None) + l1_ratio : float, default=None The Elastic-Net mixing parameter, with ``0 <= l1_ratio <= 1``. Only - used if ``penalty='elasticnet'`. Setting ``l1_ratio=0`` is equivalent + used if ``penalty='elasticnet'``. Setting ``l1_ratio=0`` is equivalent to using ``penalty='l2'``, while setting ``l1_ratio=1`` is equivalent to using ``penalty='l1'``. For ``0 < l1_ratio <1``, the penalty is a combination of L1 and L2. @@ -1339,17 +1165,17 @@ class LogisticRegression(BaseEstimator, LinearClassifierMixin, Attributes ---------- - classes_ : array, shape (n_classes, ) + classes_ : ndarray of shape (n_classes, ) A list of class labels known to the classifier. - coef_ : array, shape (1, n_features) or (n_classes, n_features) + coef_ : ndarray of shape (1, n_features) or (n_classes, n_features) Coefficient of the features in the decision function. `coef_` is of shape (1, n_features) when the given problem is binary. In particular, when `multi_class='multinomial'`, `coef_` corresponds to outcome 1 (True) and `-coef_` corresponds to outcome 0 (False). - intercept_ : array, shape (1,) or (n_classes,) + intercept_ : ndarray of shape (1,) or (n_classes,) Intercept (a.k.a. bias) added to the decision function. If `fit_intercept` is set to False, the intercept is set to zero. @@ -1358,7 +1184,7 @@ class LogisticRegression(BaseEstimator, LinearClassifierMixin, corresponds to outcome 1 (True) and `-intercept_` corresponds to outcome 0 (False). - n_iter_ : array, shape (n_classes,) or (1, ) + n_iter_ : ndarray of shape (n_classes,) or (1, ) Actual number of iterations for all classes. If binary or multinomial, it returns only 1 element. For liblinear solver, only the maximum number of iteration across all classes is given. @@ -1452,14 +1278,14 @@ def fit(self, X, y, sample_weight=None): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training vector, where n_samples is the number of samples and n_features is the number of features. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) Target vector relative to X. - sample_weight : array-like, shape (n_samples,) optional + sample_weight : array-like of shape (n_samples,) default=None Array of weights that are assigned to individual samples. If not provided, then each sample is given unit weight. @@ -1513,11 +1339,11 @@ def fit(self, X, y, sample_weight=None): else: _dtype = [np.float64, np.float32] - X, y = check_X_y(X, y, accept_sparse='csr', dtype=_dtype, order="C", - accept_large_sparse=solver != 'liblinear') + X, y = self._validate_data(X, y, accept_sparse='csr', dtype=_dtype, + order="C", + accept_large_sparse=solver != 'liblinear') check_classification_targets(y) self.classes_ = np.unique(y) - n_samples, n_features = X.shape multi_class = _check_multi_class(self.multi_class, solver, len(self.classes_)) @@ -1593,6 +1419,7 @@ def fit(self, X, y, sample_weight=None): fold_coefs_, _, n_iter_ = zip(*fold_coefs_) self.n_iter_ = np.asarray(n_iter_, dtype=np.int32)[:, 0] + n_features = X.shape[1] if multi_class == 'multinomial': self.coef_ = fold_coefs_[0][0] else: @@ -1693,18 +1520,18 @@ class LogisticRegressionCV(LogisticRegression, BaseEstimator, Parameters ---------- - Cs : list of floats or int, optional (default=10) + Cs : int or list of floats, default=10 Each of the values in Cs describes the inverse of regularization strength. If Cs is as an int, then a grid of Cs values are chosen in a logarithmic scale between 1e-4 and 1e4. Like in support vector machines, smaller values specify stronger regularization. - fit_intercept : bool, optional (default=True) + fit_intercept : bool, default=True Specifies if a constant (a.k.a. bias or intercept) should be added to the decision function. - cv : int or cross-validation generator, optional (default=None) + cv : int or cross-validation generator, default=None The default cross-validation generator used is Stratified K-Folds. If an integer is provided, then it is the number of folds used. See the module :mod:`sklearn.model_selection` module for the @@ -1713,25 +1540,25 @@ class LogisticRegressionCV(LogisticRegression, BaseEstimator, .. versionchanged:: 0.22 ``cv`` default value if None changed from 3-fold to 5-fold. - dual : bool, optional (default=False) + dual : bool, default=False Dual or primal formulation. Dual formulation is only implemented for l2 penalty with liblinear solver. Prefer dual=False when n_samples > n_features. - penalty : str, 'l1', 'l2', or 'elasticnet', optional (default='l2') + penalty : {'l1', 'l2', 'elasticnet'}, default='l2' Used to specify the norm used in the penalization. The 'newton-cg', 'sag' and 'lbfgs' solvers support only l2 penalties. 'elasticnet' is only supported by the 'saga' solver. - scoring : string, callable, or None, optional (default=None) + scoring : str or callable, default=None A string (see model evaluation documentation) or a scorer callable object / function with signature ``scorer(estimator, X, y)``. For a list of scoring functions that can be used, look at :mod:`sklearn.metrics`. The default scoring option used is 'accuracy'. - solver : str, {'newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'}, \ - optional (default='lbfgs') + solver : {'newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'}, \ + default='lbfgs' Algorithm to use in the optimization problem. @@ -1754,13 +1581,13 @@ class LogisticRegressionCV(LogisticRegression, BaseEstimator, .. versionadded:: 0.19 SAGA solver. - tol : float, optional (default=1e-4) + tol : float, default=1e-4 Tolerance for stopping criteria. - max_iter : int, optional (default=100) + max_iter : int, default=100 Maximum number of iterations of the optimization algorithm. - class_weight : dict or 'balanced', optional (default=None) + class_weight : dict or 'balanced', default=None Weights associated with classes in the form ``{class_label: weight}``. If not given, all classes are supposed to have weight one. @@ -1774,24 +1601,24 @@ class LogisticRegressionCV(LogisticRegression, BaseEstimator, .. versionadded:: 0.17 class_weight == 'balanced' - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None Number of CPU cores used during the cross-validation loop. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - verbose : int, optional (default=0) + verbose : int, default=0 For the 'liblinear', 'sag' and 'lbfgs' solvers set verbose to any positive number for verbosity. - refit : bool, optional (default=True) + refit : bool, default=True If set to True, the scores are averaged across all folds, and the coefs and the C that corresponds to the best score is taken, and a final refit is done using these parameters. Otherwise the coefs, intercepts and C that correspond to the best scores across folds are averaged. - intercept_scaling : float, optional (default=1) + intercept_scaling : float, default=1 Useful only when the solver 'liblinear' is used and self.fit_intercept is set to True. In this case, x becomes [x, self.intercept_scaling], @@ -1804,7 +1631,7 @@ class LogisticRegressionCV(LogisticRegression, BaseEstimator, To lessen the effect of regularization on synthetic feature weight (and therefore on the intercept) intercept_scaling has to be increased. - multi_class : {'ovr', 'multinomial', 'auto'}, default='auto' + multi_class : {'auto, 'ovr', 'multinomial'}, default='auto' If the option chosen is 'ovr', then a binary problem is fit for each label. For 'multinomial' the loss minimised is the multinomial loss fit across the entire probability distribution, *even when the data is @@ -1817,15 +1644,12 @@ class LogisticRegressionCV(LogisticRegression, BaseEstimator, .. versionchanged:: 0.22 Default changed from 'ovr' to 'auto' in 0.22. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. Used when `solver='sag'` or `solver='liblinear'`. + random_state : int, RandomState instance, default=None + Used when `solver='sag'`, 'saga' or 'liblinear' to shuffle the data. Note that this only applies to the solver and not the cross-validation - generator. + generator. See :term:`Glossary ` for details. - l1_ratios : list of float or None, optional (default=None) + l1_ratios : list of float, default=None The list of Elastic-Net mixing parameter, with ``0 <= l1_ratio <= 1``. Only used if ``penalty='elasticnet'``. A value of 0 is equivalent to using ``penalty='l2'``, while 1 is equivalent to using @@ -1834,30 +1658,30 @@ class LogisticRegressionCV(LogisticRegression, BaseEstimator, Attributes ---------- - classes_ : array, shape (n_classes, ) + classes_ : ndarray of shape (n_classes, ) A list of class labels known to the classifier. - coef_ : array, shape (1, n_features) or (n_classes, n_features) + coef_ : ndarray of shape (1, n_features) or (n_classes, n_features) Coefficient of the features in the decision function. `coef_` is of shape (1, n_features) when the given problem is binary. - intercept_ : array, shape (1,) or (n_classes,) + intercept_ : ndarray of shape (1,) or (n_classes,) Intercept (a.k.a. bias) added to the decision function. If `fit_intercept` is set to False, the intercept is set to zero. `intercept_` is of shape(1,) when the problem is binary. - Cs_ : array, shape (n_cs) + Cs_ : ndarray of shape (n_cs) Array of C i.e. inverse of regularization parameter values used for cross-validation. - l1_ratios_ : array, shape (n_l1_ratios) + l1_ratios_ : ndarray of shape (n_l1_ratios) Array of l1_ratios used for cross-validation. If no l1_ratio is used (i.e. penalty is not 'elasticnet'), this is set to ``[None]`` - coefs_paths_ : array, shape (n_folds, n_cs, n_features) or \ + coefs_paths_ : ndarray of shape (n_folds, n_cs, n_features) or \ (n_folds, n_cs, n_features + 1) dict with classes as the keys, and the path of coefficients obtained during cross-validating across each fold and then across each Cs @@ -1879,19 +1703,19 @@ class LogisticRegressionCV(LogisticRegression, BaseEstimator, has shape ``(n_folds, n_cs`` or ``(n_folds, n_cs, n_l1_ratios)`` if ``penalty='elasticnet'``. - C_ : array, shape (n_classes,) or (n_classes - 1,) + C_ : ndarray of shape (n_classes,) or (n_classes - 1,) Array of C that maps to the best scores across every class. If refit is set to False, then for each class, the best C is the average of the C's that correspond to the best scores for each fold. `C_` is of shape(n_classes,) when the problem is binary. - l1_ratio_ : array, shape (n_classes,) or (n_classes - 1,) + l1_ratio_ : ndarray of shape (n_classes,) or (n_classes - 1,) Array of l1_ratio that maps to the best scores across every class. If refit is set to False, then for each class, the best l1_ratio is the average of the l1_ratio's that correspond to the best scores for each fold. `l1_ratio_` is of shape(n_classes,) when the problem is binary. - n_iter_ : array, shape (n_classes, n_folds, n_cs) or (1, n_folds, n_cs) + n_iter_ : ndarray of shape (n_classes, n_folds, n_cs) or (1, n_folds, n_cs) Actual number of iterations for all classes, folds and Cs. In the binary or multinomial cases, the first dimension is equal to 1. If ``penalty='elasticnet'``, the shape is ``(n_classes, n_folds, @@ -1944,14 +1768,14 @@ def fit(self, X, y, sample_weight=None): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training vector, where n_samples is the number of samples and n_features is the number of features. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) Target vector relative to X. - sample_weight : array-like, shape (n_samples,) optional + sample_weight : array-like of shape (n_samples,) default=None Array of weights that are assigned to individual samples. If not provided, then each sample is given unit weight. @@ -1989,9 +1813,9 @@ def fit(self, X, y, sample_weight=None): "LogisticRegressionCV." ) - X, y = check_X_y(X, y, accept_sparse='csr', dtype=np.float64, - order="C", - accept_large_sparse=solver != 'liblinear') + X, y = self._validate_data(X, y, accept_sparse='csr', dtype=np.float64, + order="C", + accept_large_sparse=solver != 'liblinear') check_classification_targets(y) class_weight = self.class_weight diff --git a/sklearn/linear_model/_omp.py b/sklearn/linear_model/_omp.py index 1fc0a8b69491c..54b751423c933 100644 --- a/sklearn/linear_model/_omp.py +++ b/sklearn/linear_model/_omp.py @@ -641,7 +641,7 @@ def fit(self, X, y): self : object returns an instance of self. """ - X, y = check_X_y(X, y, multi_output=True, y_numeric=True) + X, y = self._validate_data(X, y, multi_output=True, y_numeric=True) n_features = X.shape[1] X, y, X_offset, y_offset, X_scale, Gram, Xy = \ @@ -879,8 +879,8 @@ def fit(self, X, y): self : object returns an instance of self. """ - X, y = check_X_y(X, y, y_numeric=True, ensure_min_features=2, - estimator=self) + X, y = self._validate_data(X, y, y_numeric=True, ensure_min_features=2, + estimator=self) X = as_float_array(X, copy=False, force_all_finite=False) cv = check_cv(self.cv, classifier=False) max_iter = (min(max(int(0.1 * X.shape[1]), 5), X.shape[1]) diff --git a/sklearn/linear_model/_passive_aggressive.py b/sklearn/linear_model/_passive_aggressive.py index c83a8161c3890..3b8354f5a7352 100644 --- a/sklearn/linear_model/_passive_aggressive.py +++ b/sklearn/linear_model/_passive_aggressive.py @@ -73,12 +73,11 @@ class PassiveAggressiveClassifier(BaseSGDClassifier): ``-1`` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance or None, optional, default=None - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. + random_state : int, RandomState instance, default=None + Used to shuffle the training data, when ``shuffle`` is set to + ``True``. Pass an int for reproducible output across multiple + function calls. + See :term:`Glossary `. warm_start : bool, optional When set to True, reuse the solution of the previous call to fit as @@ -131,6 +130,9 @@ class PassiveAggressiveClassifier(BaseSGDClassifier): Number of weight updates performed during training. Same as ``(n_iter_ * n_samples)``. + loss_function_ : callable + Loss function used by the algorithm. + Examples -------- >>> from sklearn.linear_model import PassiveAggressiveClassifier @@ -319,12 +321,11 @@ class PassiveAggressiveRegressor(BaseSGDRegressor): If the difference between the current prediction and the correct label is below this threshold, the model is not updated. - random_state : int, RandomState instance or None, optional, default=None - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. + random_state : int, RandomState instance, default=None + Used to shuffle the training data, when ``shuffle`` is set to + ``True``. Pass an int for reproducible output across multiple + function calls. + See :term:`Glossary `. warm_start : bool, optional When set to True, reuse the solution of the previous call to fit as diff --git a/sklearn/linear_model/_perceptron.py b/sklearn/linear_model/_perceptron.py index 10e4f27f5490e..ff50f6ebbc06e 100644 --- a/sklearn/linear_model/_perceptron.py +++ b/sklearn/linear_model/_perceptron.py @@ -12,25 +12,25 @@ class Perceptron(BaseSGDClassifier): Parameters ---------- - penalty : None, 'l2' or 'l1' or 'elasticnet' - The penalty (aka regularization term) to be used. Defaults to None. + penalty : {'l2','l1','elasticnet'}, default=None + The penalty (aka regularization term) to be used. - alpha : float + alpha : float, default=0.0001 Constant that multiplies the regularization term if regularization is - used. Defaults to 0.0001 + used. - fit_intercept : bool + fit_intercept : bool, default=True Whether the intercept should be estimated or not. If False, the - data is assumed to be already centered. Defaults to True. + data is assumed to be already centered. - max_iter : int, optional (default=1000) + max_iter : int, default=1000 The maximum number of passes over the training data (aka epochs). It only impacts the behavior in the ``fit`` method, and not the :meth:`partial_fit` method. .. versionadded:: 0.19 - tol : float or None, optional (default=1e-3) + tol : float, default=1e-3 The stopping criterion. If it is not None, the iterations will stop when (loss > previous_loss - tol). @@ -39,25 +39,24 @@ class Perceptron(BaseSGDClassifier): shuffle : bool, default=True Whether or not the training data should be shuffled after each epoch. - verbose : integer, default=0 + verbose : int, default=0 The verbosity level - eta0 : double - Constant by which the updates are multiplied. Defaults to 1. + eta0 : double, default=1 + Constant by which the updates are multiplied. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of CPUs to use to do the OVA (One Versus All, for multi-class problems) computation. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance or None, optional, default None - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. + random_state : int, RandomState instance, default=None + Used to shuffle the training data, when ``shuffle`` is set to + ``True``. Pass an int for reproducible output across multiple + function calls. + See :term:`Glossary `. early_stopping : bool, default=False Whether to use early stopping to terminate training when validation. @@ -80,7 +79,7 @@ class Perceptron(BaseSGDClassifier): .. versionadded:: 0.20 - class_weight : dict, {class_label: weight} or "balanced" or None, optional + class_weight : dict, {class_label: weight} or "balanced", default=None Preset for the class_weight fit parameter. Weights associated with classes. If not given, all classes @@ -97,18 +96,18 @@ class Perceptron(BaseSGDClassifier): Attributes ---------- - coef_ : array, shape = [1, n_features] if n_classes == 2 else [n_classes,\ - n_features] + coef_ : ndarray of shape = [1, n_features] if n_classes == 2 else \ + [n_classes, n_features] Weights assigned to the features. - intercept_ : array, shape = [1] if n_classes == 2 else [n_classes] + intercept_ : ndarray of shape = [1] if n_classes == 2 else [n_classes] Constants in decision function. n_iter_ : int The actual number of iterations to reach the stopping criterion. For multiclass fits, it is the maximum over every binary fit. - classes_ : array of shape (n_classes,) + classes_ : ndarray of shape (n_classes,) The unique classes labels. t_ : int @@ -144,6 +143,7 @@ class Perceptron(BaseSGDClassifier): https://en.wikipedia.org/wiki/Perceptron and references therein. """ + def __init__(self, penalty=None, alpha=0.0001, fit_intercept=True, max_iter=1000, tol=1e-3, shuffle=True, verbose=0, eta0=1.0, n_jobs=None, random_state=0, early_stopping=False, diff --git a/sklearn/linear_model/_ransac.py b/sklearn/linear_model/_ransac.py index 5fe50b5a21acb..cd5e3db49842d 100644 --- a/sklearn/linear_model/_ransac.py +++ b/sklearn/linear_model/_ransac.py @@ -149,11 +149,10 @@ class RANSACRegressor(MetaEstimatorMixin, RegressorMixin, If the loss on a sample is greater than the ``residual_threshold``, then this sample is classified as an outlier. - random_state : int, RandomState instance or None, optional, default None - The generator used to initialize the centers. If int, random_state is - the seed used by the random number generator; If RandomState instance, - random_state is the random number generator; If None, the random number - generator is the RandomState instance used by `np.random`. + random_state : int, RandomState instance, default=None + The generator used to initialize the centers. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Attributes ---------- @@ -247,7 +246,7 @@ def fit(self, X, y, sample_weight=None): `max_trials` randomly chosen sub-samples. """ - X = check_array(X, accept_sparse='csr') + X = self._validate_data(X, accept_sparse='csr') y = check_array(y, ensure_2d=False) check_consistent_length(X, y) @@ -320,13 +319,15 @@ def fit(self, X, y, sample_weight=None): raise ValueError("%s does not support sample_weight. Samples" " weights are only used for the calibration" " itself." % estimator_name) - sample_weight = _check_sample_weight(sample_weight, X) + if sample_weight is not None: + sample_weight = _check_sample_weight(sample_weight, X) n_inliers_best = 1 score_best = -np.inf inlier_mask_best = None X_inlier_best = None y_inlier_best = None + inlier_best_idxs_subset = None self.n_skips_no_inliers_ = 0 self.n_skips_invalid_data_ = 0 self.n_skips_invalid_model_ = 0 @@ -403,6 +404,7 @@ def fit(self, X, y, sample_weight=None): inlier_mask_best = inlier_mask_subset X_inlier_best = X_inlier_subset y_inlier_best = y_inlier_subset + inlier_best_idxs_subset = inlier_idxs_subset max_trials = min( max_trials, @@ -440,7 +442,13 @@ def fit(self, X, y, sample_weight=None): ConvergenceWarning) # estimate final model using all inliers - base_estimator.fit(X_inlier_best, y_inlier_best) + if sample_weight is None: + base_estimator.fit(X_inlier_best, y_inlier_best) + else: + base_estimator.fit( + X_inlier_best, + y_inlier_best, + sample_weight=sample_weight[inlier_best_idxs_subset]) self.estimator_ = base_estimator self.inlier_mask_ = inlier_mask_best diff --git a/sklearn/linear_model/_ridge.py b/sklearn/linear_model/_ridge.py index d217e0d832d2b..c40f641df4b5e 100644 --- a/sklearn/linear_model/_ridge.py +++ b/sklearn/linear_model/_ridge.py @@ -19,7 +19,7 @@ from ._base import LinearClassifierMixin, LinearModel, _rescale_data from ._sag import sag_solver -from ..base import RegressorMixin, MultiOutputMixin +from ..base import RegressorMixin, MultiOutputMixin, is_classifier from ..utils.extmath import safe_sparse_dot from ..utils.extmath import row_norms from ..utils import check_X_y @@ -134,7 +134,7 @@ def _solve_lsqr(X, y, alpha, max_iter=None, tol=1e-3): def _solve_cholesky(X, y, alpha): # w = inv(X^t X + alpha*Id) * X.T y - n_samples, n_features = X.shape + n_features = X.shape[1] n_targets = y.shape[1] A = safe_sparse_dot(X.T, X, dense_output=True) @@ -245,37 +245,38 @@ def ridge_regression(X, y, alpha, sample_weight=None, solver='auto', Parameters ---------- - X : {array-like, sparse matrix, LinearOperator}, - shape = [n_samples, n_features] + X : {ndarray, sparse matrix, LinearOperator} of shape \ + (n_samples, n_features) Training data - y : array-like of shape (n_samples,) or (n_samples, n_targets) + y : ndarray of shape (n_samples,) or (n_samples, n_targets) Target values - alpha : {float, array-like}, - shape = [n_targets] if array-like + alpha : float or array-like of shape (n_targets,) Regularization strength; must be a positive float. Regularization improves the conditioning of the problem and reduces the variance of the estimates. Larger values specify stronger regularization. - Alpha corresponds to ``C^-1`` in other linear models such as - LogisticRegression or LinearSVC. If an array is passed, penalties are + Alpha corresponds to ``1 / (2C)`` in other linear models such as + :class:`~sklearn.linear_model.LogisticRegression` or + :class:`sklearn.svm.LinearSVC`. If an array is passed, penalties are assumed to be specific to the targets. Hence they must correspond in number. - sample_weight : float or numpy array of shape (n_samples,), default=None - Individual weights for each sample. If sample_weight is not None and + sample_weight : float or array-like of shape (n_samples,), default=None + Individual weights for each sample. If given a float, every sample + will have the same weight. If sample_weight is not None and solver='auto', the solver will be set to 'cholesky'. .. versionadded:: 0.17 - solver : {'auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'} + solver : {'auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'}, \ + default='auto' Solver to use in the computational routines: - 'auto' chooses the solver automatically based on the type of data. - 'svd' uses a Singular Value Decomposition of X to compute the Ridge - coefficients. More stable for singular matrices than - 'cholesky'. + coefficients. More stable for singular matrices than 'cholesky'. - 'cholesky' uses the standard scipy.linalg.solve function to obtain a closed-form solution via a Cholesky decomposition of @@ -300,7 +301,7 @@ def ridge_regression(X, y, alpha, sample_weight=None, solver='auto', All last five solvers support both dense and sparse data. However, only - 'sag' and 'sparse_cg' supports sparse input when`fit_intercept` is + 'sag' and 'sparse_cg' supports sparse input when `fit_intercept` is True. .. versionadded:: 0.17 @@ -308,7 +309,7 @@ def ridge_regression(X, y, alpha, sample_weight=None, solver='auto', .. versionadded:: 0.19 SAGA solver. - max_iter : int, optional + max_iter : int, default=None Maximum number of iterations for conjugate gradient solver. For the 'sparse_cg' and 'lsqr' solvers, the default value is determined by scipy.sparse.linalg. For 'sag' and saga solver, the default value is @@ -321,12 +322,9 @@ def ridge_regression(X, y, alpha, sample_weight=None, solver='auto', Verbosity level. Setting verbose > 0 will display additional information depending on the solver used. - random_state : int, RandomState instance or None, optional, default None - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. Used when ``solver`` == 'sag'. + random_state : int, RandomState instance, default=None + Used when ``solver`` == 'sag' or 'saga' to shuffle the data. + See :term:`Glossary ` for details. return_n_iter : bool, default=False If True, the method also returns `n_iter`, the actual number of @@ -349,14 +347,14 @@ def ridge_regression(X, y, alpha, sample_weight=None, solver='auto', Returns ------- - coef : array, shape = [n_features] or [n_targets, n_features] + coef : ndarray of shape (n_features,) or (n_targets, n_features) Weight vector(s). n_iter : int, optional The actual number of iteration performed by the solver. Only returned if `return_n_iter` is True. - intercept : float or array, shape = [n_targets] + intercept : float or ndarray of shape (n_targets,) The intercept of the model. Only returned if `return_intercept` is True and if X is a scipy sparse array. @@ -364,7 +362,6 @@ def ridge_regression(X, y, alpha, sample_weight=None, solver='auto', ----- This function won't compute the intercept. """ - return _ridge_regression(X, y, alpha, sample_weight=sample_weight, solver=solver, @@ -488,8 +485,7 @@ def _ridge_regression(X, y, alpha, sample_weight=None, solver='auto', coef_, n_iter_, _ = sag_solver( X, target.ravel(), sample_weight, 'squared', alpha_i, 0, max_iter, tol, verbose, random_state, False, max_squared_sum, - init, - is_saga=solver == 'saga') + init, is_saga=solver == 'saga') if return_intercept: coef[i] = coef_[:-1] intercept[i] = coef_[-1] @@ -541,10 +537,10 @@ def fit(self, X, y, sample_weight=None): _dtype = [np.float64, np.float32] _accept_sparse = _get_valid_accept_sparse(sparse.issparse(X), self.solver) - X, y = check_X_y(X, y, - accept_sparse=_accept_sparse, - dtype=_dtype, - multi_output=True, y_numeric=True) + X, y = self._validate_data(X, y, + accept_sparse=_accept_sparse, + dtype=_dtype, + multi_output=True, y_numeric=True) if sparse.issparse(X) and self.fit_intercept: if self.solver not in ['auto', 'sparse_cg', 'sag']: raise ValueError( @@ -566,9 +562,9 @@ def fit(self, X, y, sample_weight=None): else: solver = self.solver - if ((sample_weight is not None) and - np.asarray(sample_weight).ndim > 1): - raise ValueError("Sample weights must be 1D array or scalar") + if sample_weight is not None: + sample_weight = _check_sample_weight(sample_weight, X, + dtype=X.dtype) # when X is sparse we only remove offset from y X, y, X_offset, y_offset, X_scale = self._preprocess_data( @@ -578,7 +574,7 @@ def fit(self, X, y, sample_weight=None): if solver == 'sag' and sparse.issparse(X) and self.fit_intercept: self.coef_, self.n_iter_, self.intercept_ = _ridge_regression( X, y, alpha=self.alpha, sample_weight=sample_weight, - max_iter=self.max_iter, tol=self.tol, solver=self.solver, + max_iter=self.max_iter, tol=self.tol, solver='sag', random_state=self.random_state, return_n_iter=True, return_intercept=True, check_input=False) # add the offset which was subtracted by _preprocess_data @@ -613,18 +609,19 @@ class Ridge(MultiOutputMixin, RegressorMixin, _BaseRidge): the linear least squares function and regularization is given by the l2-norm. Also known as Ridge Regression or Tikhonov regularization. This estimator has built-in support for multi-variate regression - (i.e., when y is a 2d-array of shape [n_samples, n_targets]). + (i.e., when y is a 2d-array of shape (n_samples, n_targets)). Read more in the :ref:`User Guide `. Parameters ---------- - alpha : {float, array-like of shape (n_targets,)}, default=1.0 + alpha : {float, ndarray of shape (n_targets,)}, default=1.0 Regularization strength; must be a positive float. Regularization improves the conditioning of the problem and reduces the variance of the estimates. Larger values specify stronger regularization. - Alpha corresponds to ``C^-1`` in other linear models such as - LogisticRegression or LinearSVC. If an array is passed, penalties are + Alpha corresponds to ``1 / (2C)`` in other linear models such as + :class:`~sklearn.linear_model.LogisticRegression` or + :class:`sklearn.svm.LinearSVC`. If an array is passed, penalties are assumed to be specific to the targets. Hence they must correspond in number. @@ -644,7 +641,7 @@ class Ridge(MultiOutputMixin, RegressorMixin, _BaseRidge): copy_X : bool, default=True If True, X will be copied; else, it may be overwritten. - max_iter : int, optional + max_iter : int, default=None Maximum number of iterations for conjugate gradient solver. For 'sparse_cg' and 'lsqr' solvers, the default value is determined by scipy.sparse.linalg. For 'sag' solver, the default value is 1000. @@ -652,14 +649,14 @@ class Ridge(MultiOutputMixin, RegressorMixin, _BaseRidge): tol : float, default=1e-3 Precision of the solution. - solver : {'auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'} + solver : {'auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'}, \ + default='auto' Solver to use in the computational routines: - 'auto' chooses the solver automatically based on the type of data. - 'svd' uses a Singular Value Decomposition of X to compute the Ridge - coefficients. More stable for singular matrices than - 'cholesky'. + coefficients. More stable for singular matrices than 'cholesky'. - 'cholesky' uses the standard scipy.linalg.solve function to obtain a closed-form solution. @@ -682,33 +679,31 @@ class Ridge(MultiOutputMixin, RegressorMixin, _BaseRidge): scaler from sklearn.preprocessing. All last five solvers support both dense and sparse data. However, only - 'sparse_cg' supports sparse input when `fit_intercept` is True. + 'sag' and 'sparse_cg' supports sparse input when `fit_intercept` is + True. .. versionadded:: 0.17 Stochastic Average Gradient descent solver. .. versionadded:: 0.19 SAGA solver. - random_state : int, RandomState instance or None, optional, default None - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. Used when ``solver`` == 'sag'. + random_state : int, RandomState instance, default=None + Used when ``solver`` == 'sag' or 'saga' to shuffle the data. + See :term:`Glossary ` for details. .. versionadded:: 0.17 - *random_state* to support Stochastic Average Gradient. + `random_state` to support Stochastic Average Gradient. Attributes ---------- - coef_ : array, shape (n_features,) or (n_targets, n_features) + coef_ : ndarray of shape (n_features,) or (n_targets, n_features) Weight vector(s). - intercept_ : float | array, shape = (n_targets,) + intercept_ : float or ndarray of shape (n_targets,) Independent term in decision function. Set to 0.0 if ``fit_intercept = False``. - n_iter_ : array or None, shape (n_targets,) + n_iter_ : None or ndarray of shape (n_targets,) Actual number of iterations for each target. Available only for sag and lsqr solvers. Other solvers will return None. @@ -732,8 +727,8 @@ class Ridge(MultiOutputMixin, RegressorMixin, _BaseRidge): >>> clf = Ridge(alpha=1.0) >>> clf.fit(X, y) Ridge() - """ + def __init__(self, alpha=1.0, fit_intercept=True, normalize=False, copy_X=True, max_iter=None, tol=1e-3, solver="auto", random_state=None): @@ -744,18 +739,19 @@ def __init__(self, alpha=1.0, fit_intercept=True, normalize=False, random_state=random_state) def fit(self, X, y, sample_weight=None): - """Fit Ridge regression model + """Fit Ridge regression model. Parameters ---------- - X : {array-like, sparse matrix} of shape (n_samples, n_features) + X : {ndarray, sparse matrix} of shape (n_samples, n_features) Training data - y : array-like of shape (n_samples,) or (n_samples, n_targets) + y : ndarray of shape (n_samples,) or (n_samples, n_targets) Target values - sample_weight : float or numpy array of shape [n_samples] - Individual weights for each sample + sample_weight : float or ndarray of shape (n_samples,), default=None + Individual weights for each sample. If given a float, every sample + will have the same weight. Returns ------- @@ -779,8 +775,9 @@ class RidgeClassifier(LinearClassifierMixin, _BaseRidge): Regularization strength; must be a positive float. Regularization improves the conditioning of the problem and reduces the variance of the estimates. Larger values specify stronger regularization. - Alpha corresponds to ``C^-1`` in other linear models such as - LogisticRegression or LinearSVC. + Alpha corresponds to ``1 / (2C)`` in other linear models such as + :class:`~sklearn.linear_model.LogisticRegression` or + :class:`sklearn.svm.LinearSVC`. fit_intercept : bool, default=True Whether to calculate the intercept for this model. If set to false, no @@ -798,14 +795,14 @@ class RidgeClassifier(LinearClassifierMixin, _BaseRidge): copy_X : bool, default=True If True, X will be copied; else, it may be overwritten. - max_iter : int, optional + max_iter : int, default=None Maximum number of iterations for conjugate gradient solver. The default value is determined by scipy.sparse.linalg. tol : float, default=1e-3 Precision of the solution. - class_weight : dict or 'balanced', optional + class_weight : dict or 'balanced', default=None Weights associated with classes in the form ``{class_label: weight}``. If not given, all classes are supposed to have weight one. @@ -813,14 +810,14 @@ class RidgeClassifier(LinearClassifierMixin, _BaseRidge): weights inversely proportional to class frequencies in the input data as ``n_samples / (n_classes * np.bincount(y))``. - solver : {'auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'} + solver : {'auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'}, \ + default='auto' Solver to use in the computational routines: - 'auto' chooses the solver automatically based on the type of data. - 'svd' uses a Singular Value Decomposition of X to compute the Ridge - coefficients. More stable for singular matrices than - 'cholesky'. + coefficients. More stable for singular matrices than 'cholesky'. - 'cholesky' uses the standard scipy.linalg.solve function to obtain a closed-form solution. @@ -847,29 +844,26 @@ class RidgeClassifier(LinearClassifierMixin, _BaseRidge): .. versionadded:: 0.19 SAGA solver. - random_state : int, RandomState instance or None, default=None - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. Used when ``solver`` == 'sag'. + random_state : int, RandomState instance, default=None + Used when ``solver`` == 'sag' or 'saga' to shuffle the data. + See :term:`Glossary ` for details. Attributes ---------- - coef_ : array, shape (1, n_features) or (n_classes, n_features) + coef_ : ndarray of shape (1, n_features) or (n_classes, n_features) Coefficient of the features in the decision function. ``coef_`` is of shape (1, n_features) when the given problem is binary. - intercept_ : float | array, shape = (n_targets,) + intercept_ : float or ndarray of shape (n_targets,) Independent term in decision function. Set to 0.0 if ``fit_intercept = False``. - n_iter_ : array or None, shape (n_targets,) + n_iter_ : None or ndarray of shape (n_targets,) Actual number of iterations for each target. Available only for sag and lsqr solvers. Other solvers will return None. - classes_ : array of shape (n_classes,) + classes_ : ndarray of shape (n_classes,) The classes labels. See Also @@ -903,18 +897,19 @@ def __init__(self, alpha=1.0, fit_intercept=True, normalize=False, self.class_weight = class_weight def fit(self, X, y, sample_weight=None): - """Fit Ridge regression model. + """Fit Ridge classifier model. Parameters ---------- - X : {array-like, sparse matrix} of shape (n_samples, n_features) + X : {ndarray, sparse matrix} of shape (n_samples, n_features) Training data. - y : array-like of shape (n_samples,) + y : ndarray of shape (n_samples,) Target values. - sample_weight : {float, array-like of shape (n_samples,)}, default=None - Sample weight. + sample_weight : float or ndarray of shape (n_samples,), default=None + Individual weights for each sample. If given a float, every sample + will have the same weight. .. versionadded:: 0.17 *sample_weight* support to Classifier. @@ -926,7 +921,9 @@ def fit(self, X, y, sample_weight=None): """ _accept_sparse = _get_valid_accept_sparse(sparse.issparse(X), self.solver) - check_X_y(X, y, accept_sparse=_accept_sparse, multi_output=True) + X, y = self._validate_data(X, y, accept_sparse=_accept_sparse, + multi_output=True, y_numeric=False) + sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype) self._label_binarizer = LabelBinarizer(pos_label=1, neg_label=-1) Y = self._label_binarizer.fit_transform(y) @@ -939,8 +936,6 @@ def fit(self, X, y, sample_weight=None): self.__class__.__name__)) if self.class_weight: - if sample_weight is None: - sample_weight = 1. # modify the sample weights with the corresponding class weight sample_weight = (sample_weight * compute_sample_weight(self.class_weight, y)) @@ -976,10 +971,10 @@ def _find_smallest_angle(query, vectors): Parameters ---------- - query : ndarray, shape (n_samples,) + query : ndarray of shape (n_samples,) Normalized query vector. - vectors : ndarray, shape (n_samples, n_features) + vectors : ndarray of shape (n_samples, n_features) Vectors to which we compare query, as columns. Must be normalized. """ abs_cosine = np.abs(query.dot(vectors)) @@ -1053,8 +1048,31 @@ def _matmat(self, v): return res +class _IdentityRegressor: + """Fake regressor which will directly output the prediction.""" + + def decision_function(self, y_predict): + return y_predict + + def predict(self, y_predict): + return y_predict + + +class _IdentityClassifier(LinearClassifierMixin): + """Fake classifier which will directly output the prediction. + + We inherit from LinearClassifierMixin to get the proper shape for the + output `y`. + """ + def __init__(self, classes): + self.classes_ = classes + + def decision_function(self, y_predict): + return y_predict + + class _RidgeGCV(LinearModel): - """Ridge regression with built-in Generalized Cross-Validation + """Ridge regression with built-in Generalized Cross-Validation. It allows efficient Leave-One-Out cross-validation. @@ -1086,6 +1104,10 @@ class _RidgeGCV(LinearModel): looe = y - loov = c / diag(G^-1) + The best score (negative mean squared error or user-provided scoring) is + stored in the `best_score_` attribute, and the selected hyperparameter in + `alpha_`. + References ---------- http://cbcl.mit.edu/publications/ps/MIT-CSAIL-TR-2007-025.pdf @@ -1095,7 +1117,8 @@ class _RidgeGCV(LinearModel): def __init__(self, alphas=(0.1, 1.0, 10.0), fit_intercept=True, normalize=False, scoring=None, copy_X=True, - gcv_mode=None, store_cv_values=False): + gcv_mode=None, store_cv_values=False, + is_clf=False): self.alphas = np.asarray(alphas) self.fit_intercept = fit_intercept self.normalize = normalize @@ -1103,12 +1126,15 @@ def __init__(self, alphas=(0.1, 1.0, 10.0), self.copy_X = copy_X self.gcv_mode = gcv_mode self.store_cv_values = store_cv_values + self.is_clf = is_clf - def _decomp_diag(self, v_prime, Q): + @staticmethod + def _decomp_diag(v_prime, Q): # compute diagonal of the matrix: dot(Q, dot(diag(v_prime), Q^T)) return (v_prime * Q ** 2).sum(axis=-1) - def _diag_dot(self, D, B): + @staticmethod + def _diag_dot(D, B): # compute dot(diag(D), B) if len(B.shape) > 1: # handle case where B is > 1-d @@ -1120,17 +1146,17 @@ def _compute_gram(self, X, sqrt_sw): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {ndarray, sparse matrix} of shape (n_samples, n_features) The preprocessed design matrix. - sqrt_sw : ndarray, shape (n_samples,) + sqrt_sw : ndarray of shape (n_samples,) square roots of sample weights Returns ------- - gram : ndarray, shape (n_samples, n_samples) + gram : ndarray of shape (n_samples, n_samples) The Gram matrix. - X_mean : ndarray, shape (n_feature,) + X_mean : ndarray of shape (n_feature,) The weighted mean of ``X`` for each feature. Notes @@ -1170,17 +1196,17 @@ def _compute_covariance(self, X, sqrt_sw): Parameters ---------- - X : sparse matrix, shape (n_samples, n_features) + X : sparse matrix of shape (n_samples, n_features) The preprocessed design matrix. - sqrt_sw : ndarray, shape (n_samples,) + sqrt_sw : ndarray of shape (n_samples,) square roots of sample weights Returns ------- - covariance : ndarray, shape (n_features, n_features) + covariance : ndarray of shape (n_features, n_features) The covariance matrix. - X_mean : ndarray, shape (n_feature,) + X_mean : ndarray of shape (n_feature,) The weighted mean of ``X`` for each feature. Notes @@ -1219,16 +1245,16 @@ def _sparse_multidot_diag(self, X, A, X_mean, sqrt_sw): ---------- X : sparse matrix of shape (n_samples, n_features) - A : np.ndarray, shape = (n_features, n_features) + A : ndarray of shape (n_features, n_features) - X_mean : np.ndarray, shape = (n_features,) + X_mean : ndarray of shape (n_features,) - sqrt_sw : np.ndarray, shape = (n_features,) + sqrt_sw : ndarray of shape (n_features,) square roots of sample weights Returns ------- - diag : np.ndarray, shape = (n_samples,) + diag : np.ndarray, shape (n_samples,) The computed diagonal. """ intercept_col = scale = sqrt_sw @@ -1249,7 +1275,7 @@ def _sparse_multidot_diag(self, X, A, X_mean, sqrt_sw): return diag def _eigen_decompose_gram(self, X, y, sqrt_sw): - """Eigendecomposition of X.X^T, used when n_samples <= n_features""" + """Eigendecomposition of X.X^T, used when n_samples <= n_features.""" # if X is dense it has already been centered in preprocessing K, X_mean = self._compute_gram(X, sqrt_sw) if self.fit_intercept: @@ -1263,7 +1289,7 @@ def _eigen_decompose_gram(self, X, y, sqrt_sw): return X_mean, eigvals, Q, QT_y def _solve_eigen_gram(self, alpha, y, sqrt_sw, X_mean, eigvals, Q, QT_y): - """Compute dual coefficients and diagonal of G^-1 + """Compute dual coefficients and diagonal of G^-1. Used when we have a decomposition of X.X^T (n_samples <= n_features). """ @@ -1303,7 +1329,7 @@ def _eigen_decompose_covariance(self, X, y, sqrt_sw): cov[-1] = 0 cov[:, -1] = 0 cov[-1, -1] = sqrt_sw.dot(sqrt_sw) - nullspace_dim = max(0, X.shape[1] - X.shape[0]) + nullspace_dim = max(0, n_features - n_samples) eigvals, V = linalg.eigh(cov) # remove eigenvalues and vectors in the null space of X^T.X eigvals = eigvals[nullspace_dim:] @@ -1329,7 +1355,7 @@ def _solve_eigen_covariance_no_intercept( def _solve_eigen_covariance_intercept( self, alpha, y, sqrt_sw, X_mean, eigvals, V, X): - """Compute dual coefficients and diagonal of G^-1 + """Compute dual coefficients and diagonal of G^-1. Used when we have a decomposition of X^T.X (n_samples > n_features and X is sparse), @@ -1359,7 +1385,7 @@ def _solve_eigen_covariance_intercept( def _solve_eigen_covariance( self, alpha, y, sqrt_sw, X_mean, eigvals, V, X): - """Compute dual coefficients and diagonal of G^-1 + """Compute dual coefficients and diagonal of G^-1. Used when we have a decomposition of X^T.X (n_samples > n_features and X is sparse). @@ -1386,7 +1412,7 @@ def _svd_decompose_design_matrix(self, X, y, sqrt_sw): def _solve_svd_design_matrix( self, alpha, y, sqrt_sw, X_mean, singvals_sq, U, UT_y): - """Compute dual coefficients and diagonal of G^-1 + """Compute dual coefficients and diagonal of G^-1. Used when we have an SVD decomposition of X (n_samples > n_features and X is dense). @@ -1406,36 +1432,37 @@ def _solve_svd_design_matrix( return G_inverse_diag, c def fit(self, X, y, sample_weight=None): - """Fit Ridge regression model + """Fit Ridge regression model with gcv. Parameters ---------- - X : {array-like, sparse matrix} of shape (n_samples, n_features) - Training data. Will be cast to float64 if necessary + X : {ndarray, sparse matrix} of shape (n_samples, n_features) + Training data. Will be cast to float64 if necessary. - y : array-like of shape (n_samples,) or (n_samples, n_targets) - Target values. Will be cast to float64 if necessary + y : ndarray of shape (n_samples,) or (n_samples, n_targets) + Target values. Will be cast to float64 if necessary. - sample_weight : float or array-like of shape [n_samples] - Sample weight + sample_weight : float or ndarray of shape (n_samples,), default=None + Individual weights for each sample. If given a float, every sample + will have the same weight. Returns ------- self : object """ - X, y = check_X_y(X, y, ['csr', 'csc', 'coo'], - dtype=[np.float64], - multi_output=True, y_numeric=True) + X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc', 'coo'], + dtype=[np.float64], + multi_output=True, y_numeric=True) + + if sample_weight is not None: + sample_weight = _check_sample_weight(sample_weight, X, + dtype=X.dtype) if np.any(self.alphas <= 0): raise ValueError( "alphas must be positive. Got {} containing some " "negative or null value instead.".format(self.alphas)) - sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype) - - n_samples, n_features = X.shape - X, y, X_offset, y_offset, X_scale = LinearModel._preprocess_data( X, y, self.fit_intercept, self.normalize, self.copy_X, sample_weight=sample_weight) @@ -1453,49 +1480,57 @@ def fit(self, X, y, sample_weight=None): decompose = self._svd_decompose_design_matrix solve = self._solve_svd_design_matrix + n_samples = X.shape[0] + if sample_weight is not None: X, y = _rescale_data(X, y, sample_weight) sqrt_sw = np.sqrt(sample_weight) else: - sqrt_sw = np.ones(X.shape[0], dtype=X.dtype) + sqrt_sw = np.ones(n_samples, dtype=X.dtype) + + X_mean, *decomposition = decompose(X, y, sqrt_sw) scorer = check_scoring(self, scoring=self.scoring, allow_none=True) error = scorer is None n_y = 1 if len(y.shape) == 1 else y.shape[1] - cv_values = np.zeros((n_samples * n_y, len(self.alphas)), - dtype=X.dtype) - C = [] - X_mean, *decomposition = decompose(X, y, sqrt_sw) + + if self.store_cv_values: + self.cv_values_ = np.empty( + (n_samples * n_y, len(self.alphas)), dtype=X.dtype) + + best_coef, best_score, best_alpha = None, None, None + for i, alpha in enumerate(self.alphas): G_inverse_diag, c = solve( float(alpha), y, sqrt_sw, X_mean, *decomposition) if error: squared_errors = (c / G_inverse_diag) ** 2 - cv_values[:, i] = squared_errors.ravel() + alpha_score = -squared_errors.mean() + if self.store_cv_values: + self.cv_values_[:, i] = squared_errors.ravel() else: predictions = y - (c / G_inverse_diag) - cv_values[:, i] = predictions.ravel() - C.append(c) - - if error: - best = cv_values.mean(axis=0).argmin() - else: - # The scorer want an object that will make the predictions but - # they are already computed efficiently by _RidgeGCV. This - # identity_estimator will just return them - def identity_estimator(): - pass - identity_estimator.decision_function = lambda y_predict: y_predict - identity_estimator.predict = lambda y_predict: y_predict - - # signature of scorer is (estimator, X, y) - out = [scorer(identity_estimator, cv_values[:, i], y.ravel()) - for i in range(len(self.alphas))] - best = np.argmax(out) - - self.alpha_ = self.alphas[best] - self.dual_coef_ = C[best] + if self.store_cv_values: + self.cv_values_[:, i] = predictions.ravel() + + if self.is_clf: + identity_estimator = _IdentityClassifier( + classes=np.arange(n_y) + ) + predictions_, y_ = predictions, y.argmax(axis=1) + else: + identity_estimator = _IdentityRegressor() + predictions_, y_ = predictions.ravel(), y.ravel() + + alpha_score = scorer(identity_estimator, predictions_, y_) + + if (best_score is None) or (alpha_score > best_score): + best_coef, best_score, best_alpha = c, alpha_score, alpha + + self.alpha_ = best_alpha + self.best_score_ = best_score + self.dual_coef_ = best_coef self.coef_ = safe_sparse_dot(self.dual_coef_.T, X) X_offset += X_mean * X_scale @@ -1506,7 +1541,7 @@ def identity_estimator(): cv_values_shape = n_samples, len(self.alphas) else: cv_values_shape = n_samples, n_y, len(self.alphas) - self.cv_values_ = cv_values.reshape(cv_values_shape) + self.cv_values_ = self.cv_values_.reshape(cv_values_shape) return self @@ -1525,19 +1560,20 @@ def __init__(self, alphas=(0.1, 1.0, 10.0), self.store_cv_values = store_cv_values def fit(self, X, y, sample_weight=None): - """Fit Ridge regression model + """Fit Ridge regression model with cv. Parameters ---------- - X : array-like of shape (n_samples, n_features) + X : ndarray of shape (n_samples, n_features) Training data. If using GCV, will be cast to float64 if necessary. - y : array-like of shape (n_samples,) or (n_samples, n_targets) - Target values. Will be cast to X's dtype if necessary + y : ndarray of shape (n_samples,) or (n_samples, n_targets) + Target values. Will be cast to X's dtype if necessary. - sample_weight : float or array-like of shape [n_samples] - Sample weight + sample_weight : float or ndarray of shape (n_samples,), default=None + Individual weights for each sample. If given a float, every sample + will have the same weight. Returns ------- @@ -1558,9 +1594,11 @@ def fit(self, X, y, sample_weight=None): normalize=self.normalize, scoring=self.scoring, gcv_mode=self.gcv_mode, - store_cv_values=self.store_cv_values) + store_cv_values=self.store_cv_values, + is_clf=is_classifier(self)) estimator.fit(X, y, sample_weight=sample_weight) self.alpha_ = estimator.alpha_ + self.best_score_ = estimator.best_score_ if self.store_cv_values: self.cv_values_ = estimator.cv_values_ else: @@ -1569,16 +1607,19 @@ def fit(self, X, y, sample_weight=None): " are incompatible") parameters = {'alpha': self.alphas} solver = 'sparse_cg' if sparse.issparse(X) else 'auto' - gs = GridSearchCV(Ridge(fit_intercept=self.fit_intercept, + model = RidgeClassifier if is_classifier(self) else Ridge + gs = GridSearchCV(model(fit_intercept=self.fit_intercept, normalize=self.normalize, solver=solver), parameters, cv=cv, scoring=self.scoring) gs.fit(X, y, sample_weight=sample_weight) estimator = gs.best_estimator_ self.alpha_ = gs.best_estimator_.alpha + self.best_score_ = gs.best_score_ self.coef_ = estimator.coef_ self.intercept_ = estimator.intercept_ + self.n_features_in_ = estimator.n_features_in_ return self @@ -1595,13 +1636,14 @@ class RidgeCV(MultiOutputMixin, RegressorMixin, _BaseRidgeCV): Parameters ---------- - alphas : numpy array of shape (n_alphas,), default=(0.1, 1.0, 10.0) + alphas : ndarray of shape (n_alphas,), default=(0.1, 1.0, 10.0) Array of alpha values to try. Regularization strength; must be a positive float. Regularization improves the conditioning of the problem and reduces the variance of the estimates. Larger values specify stronger regularization. - Alpha corresponds to ``C^-1`` in other linear models such as - LogisticRegression or LinearSVC. + Alpha corresponds to ``1 / (2C)`` in other linear models such as + :class:`~sklearn.linear_model.LogisticRegression` or + :class:`sklearn.svm.LinearSVC`. If using generalized cross-validation, alphas must be positive. fit_intercept : bool, default=True @@ -1617,14 +1659,14 @@ class RidgeCV(MultiOutputMixin, RegressorMixin, _BaseRidgeCV): :class:`sklearn.preprocessing.StandardScaler` before calling ``fit`` on an estimator with ``normalize=False``. - scoring : string, callable or None, default=None + scoring : string, callable, default=None A string (see model evaluation documentation) or a scorer callable object / function with signature ``scorer(estimator, X, y)``. If None, the negative mean squared error if cv is 'auto' or None (i.e. when using generalized cross-validation), and r2 score otherwise. - cv : int, cross-validation generator or an iterable, optional + cv : int, cross-validation generator or an iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: @@ -1641,7 +1683,7 @@ class RidgeCV(MultiOutputMixin, RegressorMixin, _BaseRidgeCV): Refer :ref:`User Guide ` for the various cross-validation strategies that can be used here. - gcv_mode : {None, 'auto', 'svd', eigen'}, optional + gcv_mode : {'auto', 'svd', eigen'}, default='auto' Flag indicating which strategy to use when performing Generalized Cross-Validation. Options are:: @@ -1653,7 +1695,7 @@ class RidgeCV(MultiOutputMixin, RegressorMixin, _BaseRidgeCV): The 'auto' mode is the default and is intended to pick the cheaper option of the two depending on the shape of the training data. - store_cv_values : boolean, default=False + store_cv_values : bool, default=False Flag indicating if the cross-validation values corresponding to each alpha should be stored in the ``cv_values_`` attribute (see below). This flag is only compatible with ``cv=None`` (i.e. using @@ -1661,23 +1703,27 @@ class RidgeCV(MultiOutputMixin, RegressorMixin, _BaseRidgeCV): Attributes ---------- - cv_values_ : array, shape = [n_samples, n_alphas] or \ - shape = [n_samples, n_targets, n_alphas], optional - Cross-validation values for each alpha (if ``store_cv_values=True``\ - and ``cv=None``). After ``fit()`` has been called, this attribute \ - will contain the mean squared errors (by default) or the values \ - of the ``{loss,score}_func`` function (if provided in the constructor). - - coef_ : array, shape = [n_features] or [n_targets, n_features] + cv_values_ : ndarray of shape (n_samples, n_alphas) or \ + shape (n_samples, n_targets, n_alphas), optional + Cross-validation values for each alpha (only available if \ + ``store_cv_values=True`` and ``cv=None``). After ``fit()`` has been \ + called, this attribute will contain the mean squared errors \ + (by default) or the values of the ``{loss,score}_func`` function \ + (if provided in the constructor). + + coef_ : ndarray of shape (n_features) or (n_targets, n_features) Weight vector(s). - intercept_ : float | array, shape = (n_targets,) + intercept_ : float or ndarray of shape (n_targets,) Independent term in decision function. Set to 0.0 if ``fit_intercept = False``. alpha_ : float Estimated regularization parameter. + best_score_ : float + Score of base estimator with best alpha. + Examples -------- >>> from sklearn.datasets import load_diabetes @@ -1693,7 +1739,6 @@ class RidgeCV(MultiOutputMixin, RegressorMixin, _BaseRidgeCV): RidgeClassifier : Ridge classifier RidgeClassifierCV : Ridge classifier with built-in cross validation """ - pass class RidgeClassifierCV(LinearClassifierMixin, _BaseRidgeCV): @@ -1709,13 +1754,14 @@ class RidgeClassifierCV(LinearClassifierMixin, _BaseRidgeCV): Parameters ---------- - alphas : numpy array of shape (n_alphas,), default=(0.1, 1.0, 10.0) + alphas : ndarray of shape (n_alphas,), default=(0.1, 1.0, 10.0) Array of alpha values to try. Regularization strength; must be a positive float. Regularization improves the conditioning of the problem and reduces the variance of the estimates. Larger values specify stronger regularization. - Alpha corresponds to ``C^-1`` in other linear models such as - LogisticRegression or LinearSVC. + Alpha corresponds to ``1 / (2C)`` in other linear models such as + :class:`~sklearn.linear_model.LogisticRegression` or + :class:`sklearn.svm.LinearSVC`. fit_intercept : bool, default=True Whether to calculate the intercept for this model. If set @@ -1730,12 +1776,12 @@ class RidgeClassifierCV(LinearClassifierMixin, _BaseRidgeCV): :class:`sklearn.preprocessing.StandardScaler` before calling ``fit`` on an estimator with ``normalize=False``. - scoring : string, callable or None, default=None + scoring : string, callable, default=None A string (see model evaluation documentation) or a scorer callable object / function with signature ``scorer(estimator, X, y)``. - cv : int, cross-validation generator or an iterable, optional + cv : int, cross-validation generator or an iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: @@ -1747,7 +1793,7 @@ class RidgeClassifierCV(LinearClassifierMixin, _BaseRidgeCV): Refer :ref:`User Guide ` for the various cross-validation strategies that can be used here. - class_weight : dict or 'balanced', optional + class_weight : dict or 'balanced', default=None Weights associated with classes in the form ``{class_label: weight}``. If not given, all classes are supposed to have weight one. @@ -1755,7 +1801,7 @@ class RidgeClassifierCV(LinearClassifierMixin, _BaseRidgeCV): weights inversely proportional to class frequencies in the input data as ``n_samples / (n_classes * np.bincount(y))`` - store_cv_values : boolean, default=False + store_cv_values : bool, default=False Flag indicating if the cross-validation values corresponding to each alpha should be stored in the ``cv_values_`` attribute (see below). This flag is only compatible with ``cv=None`` (i.e. using @@ -1763,26 +1809,29 @@ class RidgeClassifierCV(LinearClassifierMixin, _BaseRidgeCV): Attributes ---------- - cv_values_ : array, shape = [n_samples, n_targets, n_alphas], optional + cv_values_ : ndarray of shape (n_samples, n_targets, n_alphas), optional Cross-validation values for each alpha (if ``store_cv_values=True`` and ``cv=None``). After ``fit()`` has been called, this attribute will contain the mean squared errors (by default) or the values of the ``{loss,score}_func`` function (if provided in the constructor). This attribute exists only when ``store_cv_values`` is True. - coef_ : array, shape (1, n_features) or (n_targets, n_features) + coef_ : ndarray of shape (1, n_features) or (n_targets, n_features) Coefficient of the features in the decision function. ``coef_`` is of shape (1, n_features) when the given problem is binary. - intercept_ : float | array, shape = (n_targets,) + intercept_ : float or ndarray of shape (n_targets,) Independent term in decision function. Set to 0.0 if ``fit_intercept = False``. alpha_ : float - Estimated regularization parameter + Estimated regularization parameter. - classes_ : array of shape (n_classes,) + best_score_ : float + Score of base estimator with best alpha. + + classes_ : ndarray of shape (n_classes,) The classes labels. Examples @@ -1816,27 +1865,29 @@ def __init__(self, alphas=(0.1, 1.0, 10.0), fit_intercept=True, self.class_weight = class_weight def fit(self, X, y, sample_weight=None): - """Fit the ridge classifier. + """Fit Ridge classifier with cv. Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : ndarray of shape (n_samples, n_features) Training vectors, where n_samples is the number of samples and n_features is the number of features. When using GCV, will be cast to float64 if necessary. - y : array-like, shape (n_samples,) - Target values. Will be cast to X's dtype if necessary + y : ndarray of shape (n_samples,) + Target values. Will be cast to X's dtype if necessary. - sample_weight : {float, array-like of shape (n_samples,)}, default=None - Sample weight. + sample_weight : float or ndarray of shape (n_samples,), default=None + Individual weights for each sample. If given a float, every sample + will have the same weight. Returns ------- self : object """ - check_X_y(X, y, accept_sparse=['csr', 'csc', 'coo'], - multi_output=True) + X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc', 'coo'], + multi_output=True, y_numeric=False) + sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype) self._label_binarizer = LabelBinarizer(pos_label=1, neg_label=-1) Y = self._label_binarizer.fit_transform(y) @@ -1844,13 +1895,12 @@ def fit(self, X, y, sample_weight=None): y = column_or_1d(y, warn=True) if self.class_weight: - if sample_weight is None: - sample_weight = 1. # modify the sample weights with the corresponding class weight sample_weight = (sample_weight * compute_sample_weight(self.class_weight, y)) - _BaseRidgeCV.fit(self, X, Y, sample_weight=sample_weight) + target = Y if self.cv is None else y + _BaseRidgeCV.fit(self, X, target, sample_weight=sample_weight) return self @property diff --git a/sklearn/linear_model/_sag.py b/sklearn/linear_model/_sag.py index c5cd88fe6710a..9fe6f076f5145 100644 --- a/sklearn/linear_model/_sag.py +++ b/sklearn/linear_model/_sag.py @@ -151,12 +151,10 @@ def sag_solver(X, y, sample_weight=None, loss='log', alpha=1., beta=0., verbose : integer, optional The verbosity level. - random_state : int, RandomState instance or None, optional, default None - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. + random_state : int, RandomState instance, default=None + Used when shuffling the data. Pass an int for reproducible output + across multiple function calls. + See :term:`Glossary `. check_input : bool, default True If False, the input arrays X and y will not be checked. diff --git a/sklearn/linear_model/_sgd_fast.pyx b/sklearn/linear_model/_sgd_fast.pyx index 545e3b6a99f1f..cc34400dbcfef 100644 --- a/sklearn/linear_model/_sgd_fast.pyx +++ b/sklearn/linear_model/_sgd_fast.pyx @@ -332,155 +332,39 @@ cdef class SquaredEpsilonInsensitive(Regression): return SquaredEpsilonInsensitive, (self.epsilon,) -def plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights, - double intercept, - LossFunction loss, - int penalty_type, - double alpha, double C, - double l1_ratio, - SequentialDataset dataset, - np.ndarray[unsigned char, ndim=1, mode='c'] validation_mask, - bint early_stopping, validation_score_cb, - int n_iter_no_change, - int max_iter, double tol, int fit_intercept, - int verbose, bint shuffle, np.uint32_t seed, - double weight_pos, double weight_neg, - int learning_rate, double eta0, - double power_t, - double t=1.0, - double intercept_decay=1.0): - """Plain SGD for generic loss functions and penalties. - - Parameters - ---------- - weights : ndarray[double, ndim=1] - The allocated coef_ vector. - intercept : double - The initial intercept. - loss : LossFunction - A concrete ``LossFunction`` object. - penalty_type : int - The penalty 2 for L2, 1 for L1, and 3 for Elastic-Net. - alpha : float - The regularization parameter. - C : float - Maximum step size for passive aggressive. - l1_ratio : float - The Elastic Net mixing parameter, with 0 <= l1_ratio <= 1. - l1_ratio=0 corresponds to L2 penalty, l1_ratio=1 to L1. - dataset : SequentialDataset - A concrete ``SequentialDataset`` object. - validation_mask : ndarray[unsigned char, ndim=1] - Equal to True on the validation set. - early_stopping : boolean - Whether to use a stopping criterion based on the validation set. - validation_score_cb : callable - A callable to compute a validation score given the current - coefficients and intercept values. - Used only if early_stopping is True. - n_iter_no_change : int - Number of iteration with no improvement to wait before stopping. - max_iter : int - The maximum number of iterations (epochs). - tol: double - The tolerance for the stopping criterion. - fit_intercept : int - Whether or not to fit the intercept (1 or 0). - verbose : int - Print verbose output; 0 for quite. - shuffle : boolean - Whether to shuffle the training data before each epoch. - weight_pos : float - The weight of the positive class. - weight_neg : float - The weight of the negative class. - seed : np.uint32_t - Seed of the pseudorandom number generator used to shuffle the data. - learning_rate : int - The learning rate: - (1) constant, eta = eta0 - (2) optimal, eta = 1.0/(alpha * t). - (3) inverse scaling, eta = eta0 / pow(t, power_t) - (4) adaptive decrease - (5) Passive Aggressive-I, eta = min(alpha, loss/norm(x)) - (6) Passive Aggressive-II, eta = 1.0 / (norm(x) + 0.5*alpha) - eta0 : double - The initial learning rate. - power_t : double - The exponent for inverse scaling learning rate. - t : double - Initial state of the learning rate. This value is equal to the - iteration count except when the learning rate is set to `optimal`. - Default: 1.0. - intercept_decay : double - The decay ratio of intercept, used in updating intercept. - - Returns - ------- - weights : array, shape=[n_features] - The fitted weight vector. - intercept : float - The fitted intercept term. - n_iter_ : int - The actual number of iter (epochs). - """ - standard_weights, standard_intercept,\ - _, _, n_iter_ = _plain_sgd(weights, - intercept, - None, - 0, - loss, - penalty_type, - alpha, C, - l1_ratio, - dataset, - validation_mask, - early_stopping, - validation_score_cb, - n_iter_no_change, - max_iter, tol, fit_intercept, - verbose, shuffle, seed, - weight_pos, weight_neg, - learning_rate, eta0, - power_t, - t, - intercept_decay, - 0) - return standard_weights, standard_intercept, n_iter_ - - -def average_sgd(np.ndarray[double, ndim=1, mode='c'] weights, - double intercept, - np.ndarray[double, ndim=1, mode='c'] average_weights, - double average_intercept, - LossFunction loss, - int penalty_type, - double alpha, double C, - double l1_ratio, - SequentialDataset dataset, - np.ndarray[unsigned char, ndim=1, mode='c'] validation_mask, - bint early_stopping, validation_score_cb, - int n_iter_no_change, - int max_iter, double tol, int fit_intercept, - int verbose, bint shuffle, np.uint32_t seed, - double weight_pos, double weight_neg, - int learning_rate, double eta0, - double power_t, - double t=1.0, - double intercept_decay=1.0, - int average=1): - """Average SGD for generic loss functions and penalties. +def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights, + double intercept, + np.ndarray[double, ndim=1, mode='c'] average_weights, + double average_intercept, + LossFunction loss, + int penalty_type, + double alpha, double C, + double l1_ratio, + SequentialDataset dataset, + np.ndarray[unsigned char, ndim=1, mode='c'] validation_mask, + bint early_stopping, validation_score_cb, + int n_iter_no_change, + int max_iter, double tol, int fit_intercept, + int verbose, bint shuffle, np.uint32_t seed, + double weight_pos, double weight_neg, + int learning_rate, double eta0, + double power_t, + double t=1.0, + double intercept_decay=1.0, + int average=0): + """SGD for generic loss functions and penalties with optional averaging Parameters ---------- weights : ndarray[double, ndim=1] - The allocated coef_ vector. + The allocated vector of weights. intercept : double The initial intercept. average_weights : ndarray[double, ndim=1] - The average weights as computed for ASGD + The average weights as computed for ASGD. Should be None if average + is 0. average_intercept : double - The average intercept for ASGD + The average intercept for ASGD. Should be 0 if average is 0. loss : LossFunction A concrete ``LossFunction`` object. penalty_type : int @@ -549,55 +433,14 @@ def average_sgd(np.ndarray[double, ndim=1, mode='c'] weights, intercept : float The fitted intercept term. average_weights : array shape=[n_features] - The averaged weights across iterations + The averaged weights across iterations. Values are valid only if + average > 0. average_intercept : float - The averaged intercept across iterations + The averaged intercept across iterations. + Values are valid only if average > 0. n_iter_ : int The actual number of iter (epochs). """ - return _plain_sgd(weights, - intercept, - average_weights, - average_intercept, - loss, - penalty_type, - alpha, C, - l1_ratio, - dataset, - validation_mask, - early_stopping, - validation_score_cb, - n_iter_no_change, - max_iter, tol, fit_intercept, - verbose, shuffle, seed, - weight_pos, weight_neg, - learning_rate, eta0, - power_t, - t, - intercept_decay, - average) - - -def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights, - double intercept, - np.ndarray[double, ndim=1, mode='c'] average_weights, - double average_intercept, - LossFunction loss, - int penalty_type, - double alpha, double C, - double l1_ratio, - SequentialDataset dataset, - np.ndarray[unsigned char, ndim=1, mode='c'] validation_mask, - bint early_stopping, validation_score_cb, - int n_iter_no_change, - int max_iter, double tol, int fit_intercept, - int verbose, bint shuffle, np.uint32_t seed, - double weight_pos, double weight_neg, - int learning_rate, double eta0, - double power_t, - double t=1.0, - double intercept_decay=1.0, - int average=0): # get the data information into easy vars cdef Py_ssize_t n_samples = dataset.n_samples diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index eb1e9e7b545e7..bca1928ecf481 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -22,7 +22,7 @@ from ..exceptions import ConvergenceWarning from ..model_selection import StratifiedShuffleSplit, ShuffleSplit -from ._sgd_fast import plain_sgd, average_sgd +from ._sgd_fast import _plain_sgd from ..utils import compute_class_weight from ._sgd_fast import Hinge from ._sgd_fast import SquaredHinge @@ -33,6 +33,7 @@ from ._sgd_fast import EpsilonInsensitive from ._sgd_fast import SquaredEpsilonInsensitive from ..utils.fixes import _joblib_parallel_args +from ..utils import deprecated LEARNING_RATE_TYPES = {"constant": 1, "optimal": 2, "invscaling": 3, "adaptive": 4, "pa1": 5, "pa2": 6} @@ -230,12 +231,12 @@ def _allocate_parameter_mem(self, n_classes, n_features, coef_init=None, # initialize average parameters if self.average > 0: - self.standard_coef_ = self.coef_ - self.standard_intercept_ = self.intercept_ - self.average_coef_ = np.zeros(self.coef_.shape, + self._standard_coef = self.coef_ + self._standard_intercept = self.intercept_ + self._average_coef = np.zeros(self.coef_.shape, dtype=np.float64, order="C") - self.average_intercept_ = np.zeros(self.standard_intercept_.shape, + self._average_intercept = np.zeros(self._standard_intercept.shape, dtype=np.float64, order="C") @@ -244,12 +245,12 @@ def _make_validation_split(self, y): Parameters ---------- - y : array, shape (n_samples, ) + y : ndarray of shape (n_samples, ) Target values. Returns ------- - validation_mask : array, shape (n_samples, ) + validation_mask : ndarray of shape (n_samples, ) Equal to 1 on the validation set, 0 on the training set. """ n_samples = y.shape[0] @@ -286,6 +287,30 @@ def _make_validation_score_cb(self, validation_mask, X, y, sample_weight, self, X[validation_mask], y[validation_mask], sample_weight[validation_mask], classes=classes) + @deprecated("Attribute standard_coef_ was deprecated " + "in version 0.23 and will be removed in 0.25.") + @property + def standard_coef_(self): + return self._standard_coef + + @deprecated("Attribute standard_intercept_ was deprecated " + "in version 0.23 and will be removed in 0.25.") + @property + def standard_intercept_(self): + return self._standard_intercept + + @deprecated("Attribute average_coef_ was deprecated " + "in version 0.23 and will be removed in 0.25.") + @property + def average_coef_(self): + return self._average_coef + + @deprecated("Attribute average_intercept_ was deprecated " + "in version 0.23 and will be removed in 0.25.") + @property + def average_intercept_(self): + return self._average_intercept + def _prepare_fit_binary(est, y, i): """Initialization for fit_binary. @@ -302,19 +327,19 @@ def _prepare_fit_binary(est, y, i): coef = est.coef_.ravel() intercept = est.intercept_[0] else: - coef = est.standard_coef_.ravel() - intercept = est.standard_intercept_[0] - average_coef = est.average_coef_.ravel() - average_intercept = est.average_intercept_[0] + coef = est._standard_coef.ravel() + intercept = est._standard_intercept[0] + average_coef = est._average_coef.ravel() + average_intercept = est._average_intercept[0] else: if not est.average: coef = est.coef_[i] intercept = est.intercept_[i] else: - coef = est.standard_coef_[i] - intercept = est.standard_intercept_[i] - average_coef = est.average_coef_[i] - average_intercept = est.average_intercept_[i] + coef = est._standard_coef[i] + intercept = est._standard_intercept[i] + average_coef = est._average_coef[i] + average_intercept = est._average_intercept[i] return y_i, coef, intercept, average_coef, average_intercept @@ -362,11 +387,11 @@ def fit_binary(est, i, X, y, alpha, C, learning_rate, max_iter, sample_weight : numpy array of shape [n_samples, ] The weight of each sample - validation_mask : numpy array of shape [n_samples, ] or None + validation_mask : numpy array of shape [n_samples, ], default=None Precomputed validation mask in case _fit_binary is called in the context of a one-vs-rest reduction. - random_state : int, RandomState instance or None, optional (default=None) + random_state : int, RandomState instance, default=None If int, random_state is the seed used by the random number generator; If RandomState instance, random_state is the random number generator; If None, the random number generator is the RandomState instance used @@ -397,39 +422,21 @@ def fit_binary(est, i, X, y, alpha, C, learning_rate, max_iter, tol = est.tol if est.tol is not None else -np.inf - if not est.average: - result = plain_sgd(coef, intercept, est.loss_function_, - penalty_type, alpha, C, est.l1_ratio, - dataset, validation_mask, est.early_stopping, - validation_score_cb, int(est.n_iter_no_change), - max_iter, tol, int(est.fit_intercept), - int(est.verbose), int(est.shuffle), seed, - pos_weight, neg_weight, - learning_rate_type, est.eta0, - est.power_t, est.t_, intercept_decay) - - else: - standard_coef, standard_intercept, average_coef, average_intercept, \ - n_iter_ = average_sgd(coef, intercept, average_coef, - average_intercept, est.loss_function_, - penalty_type, alpha, C, est.l1_ratio, - dataset, validation_mask, est.early_stopping, - validation_score_cb, - int(est.n_iter_no_change), max_iter, tol, - int(est.fit_intercept), int(est.verbose), - int(est.shuffle), seed, pos_weight, - neg_weight, learning_rate_type, est.eta0, - est.power_t, est.t_, intercept_decay, - est.average) + coef, intercept, average_coef, average_intercept, n_iter_ = _plain_sgd( + coef, intercept, average_coef, average_intercept, est.loss_function_, + penalty_type, alpha, C, est.l1_ratio, dataset, validation_mask, + est.early_stopping, validation_score_cb, int(est.n_iter_no_change), + max_iter, tol, int(est.fit_intercept), int(est.verbose), + int(est.shuffle), seed, pos_weight, neg_weight, learning_rate_type, + est.eta0, est.power_t, est.t_, intercept_decay, est.average) + if est.average: if len(est.classes_) == 2: - est.average_intercept_[0] = average_intercept + est._average_intercept[0] = average_intercept else: - est.average_intercept_[i] = average_intercept + est._average_intercept[i] = average_intercept - result = standard_coef, standard_intercept, n_iter_ - - return result + return coef, intercept, n_iter_ class BaseSGDClassifier(LinearClassifierMixin, BaseSGD, metaclass=ABCMeta): @@ -521,8 +528,9 @@ def _fit(self, X, y, alpha, C, loss, learning_rate, coef_init=None, if hasattr(self, "classes_"): self.classes_ = None - X, y = check_X_y(X, y, 'csr', dtype=np.float64, order="C", - accept_large_sparse=False) + X, y = self._validate_data(X, y, accept_sparse='csr', + dtype=np.float64, order="C", + accept_large_sparse=False) # labels can be encoded as float, int, or string literals # np.unique sorts in asc order; largest class id is positive class @@ -538,10 +546,10 @@ def _fit(self, X, y, alpha, C, loss, learning_rate, coef_init=None, self.intercept_ = None if self.average > 0: - self.standard_coef_ = self.coef_ - self.standard_intercept_ = self.intercept_ - self.average_coef_ = None - self.average_intercept_ = None + self._standard_coef = self.coef_ + self._standard_intercept = self.intercept_ + self._average_coef = None + self._average_intercept = None # Clear iteration count for multiple call to fit. self.t_ = 1.0 @@ -573,12 +581,12 @@ def _fit_binary(self, X, y, alpha, C, sample_weight, # need to be 2d if self.average > 0: if self.average <= self.t_ - 1: - self.coef_ = self.average_coef_.reshape(1, -1) - self.intercept_ = self.average_intercept_ + self.coef_ = self._average_coef.reshape(1, -1) + self.intercept_ = self._average_intercept else: - self.coef_ = self.standard_coef_.reshape(1, -1) - self.standard_intercept_ = np.atleast_1d(intercept) - self.intercept_ = self.standard_intercept_ + self.coef_ = self._standard_coef.reshape(1, -1) + self._standard_intercept = np.atleast_1d(intercept) + self.intercept_ = self._standard_intercept else: self.coef_ = coef.reshape(1, -1) # intercept is a float, need to convert it to an array of length 1 @@ -621,12 +629,12 @@ def _fit_multiclass(self, X, y, alpha, C, learning_rate, if self.average > 0: if self.average <= self.t_ - 1.0: - self.coef_ = self.average_coef_ - self.intercept_ = self.average_intercept_ + self.coef_ = self._average_coef + self.intercept_ = self._average_intercept else: - self.coef_ = self.standard_coef_ - self.standard_intercept_ = np.atleast_1d(self.intercept_) - self.intercept_ = self.standard_intercept_ + self.coef_ = self._standard_coef + self._standard_intercept = np.atleast_1d(self.intercept_) + self.intercept_ = self._standard_intercept def partial_fit(self, X, y, classes=None, sample_weight=None): """Perform one epoch of stochastic gradient descent on given samples. @@ -641,10 +649,10 @@ def partial_fit(self, X, y, classes=None, sample_weight=None): X : {array-like, sparse matrix}, shape (n_samples, n_features) Subset of the training data. - y : numpy array, shape (n_samples,) + y : ndarray of shape (n_samples,) Subset of the target values. - classes : array, shape (n_classes,) + classes : ndarray of shape (n_classes,), default=None Classes across all calls to partial_fit. Can be obtained by via `np.unique(y_all)`, where y_all is the target vector of the entire dataset. @@ -652,7 +660,7 @@ def partial_fit(self, X, y, classes=None, sample_weight=None): and can be omitted in the subsequent calls. Note that y doesn't need to contain all labels in `classes`. - sample_weight : array-like, shape (n_samples,), optional + sample_weight : array-like, shape (n_samples,), default=None Weights applied to individual samples. If not provided, uniform weights are assumed. @@ -685,16 +693,16 @@ def fit(self, X, y, coef_init=None, intercept_init=None, X : {array-like, sparse matrix}, shape (n_samples, n_features) Training data. - y : numpy array, shape (n_samples,) + y : ndarray of shape (n_samples,) Target values. - coef_init : array, shape (n_classes, n_features) + coef_init : ndarray of shape (n_classes, n_features), default=None The initial coefficients to warm-start the optimization. - intercept_init : array, shape (n_classes,) + intercept_init : ndarray of shape (n_classes,), default=None The initial intercept to warm-start the optimization. - sample_weight : array-like, shape (n_samples,), optional + sample_weight : array-like, shape (n_samples,), default=None Weights applied to individual samples. If not provided, uniform weights are assumed. These weights will be multiplied with class_weight (passed through the @@ -738,7 +746,7 @@ class SGDClassifier(BaseSGDClassifier): Parameters ---------- - loss : str, default: 'hinge' + loss : str, default='hinge' The loss function to be used. Defaults to 'hinge', which gives a linear SVM. @@ -754,42 +762,41 @@ class SGDClassifier(BaseSGDClassifier): The other losses are designed for regression but can be useful in classification as well; see SGDRegressor for a description. - penalty : str, 'none', 'l2', 'l1', or 'elasticnet' + penalty : {'l2', 'l1', 'elasticnet'}, default='l2' The penalty (aka regularization term) to be used. Defaults to 'l2' which is the standard regularizer for linear SVM models. 'l1' and 'elasticnet' might bring sparsity to the model (feature selection) not achievable with 'l2'. - alpha : float + alpha : float, default=0.0001 Constant that multiplies the regularization term. Defaults to 0.0001. Also used to compute learning_rate when set to 'optimal'. - l1_ratio : float + l1_ratio : float, default=0.15 The Elastic Net mixing parameter, with 0 <= l1_ratio <= 1. l1_ratio=0 corresponds to L2 penalty, l1_ratio=1 to L1. Defaults to 0.15. - fit_intercept : bool + fit_intercept : bool, default=True Whether the intercept should be estimated or not. If False, the data is assumed to be already centered. Defaults to True. - max_iter : int, optional (default=1000) + max_iter : int, default=1000 The maximum number of passes over the training data (aka epochs). It only impacts the behavior in the ``fit`` method, and not the :meth:`partial_fit` method. .. versionadded:: 0.19 - tol : float or None, optional (default=1e-3) + tol : float, default=1e-3 The stopping criterion. If it is not None, the iterations will stop when (loss > best_loss - tol) for ``n_iter_no_change`` consecutive epochs. .. versionadded:: 0.19 - shuffle : bool, optional + shuffle : bool, default=True Whether or not the training data should be shuffled after each epoch. - Defaults to True. verbose : int, default=0 The verbosity level. @@ -802,21 +809,19 @@ class SGDClassifier(BaseSGDClassifier): For epsilon-insensitive, any differences between the current prediction and the correct label are ignored if they are less than this threshold. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of CPUs to use to do the OVA (One Versus All, for multi-class problems) computation. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance or None, optional (default=None) - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. + random_state : int, RandomState instance, default=None + Used for shuffling the data, when ``shuffle`` is set to ``True``. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. - learning_rate : str, optional + learning_rate : str, default='optimal' The learning rate schedule: 'constant': @@ -832,12 +837,12 @@ class SGDClassifier(BaseSGDClassifier): training loss by tol or fail to increase validation score by tol if early_stopping is True, the current learning rate is divided by 5. - eta0 : double + eta0 : double, default=0.0 The initial learning rate for the 'constant', 'invscaling' or 'adaptive' schedules. The default value is 0.0 as eta0 is not used by the default schedule 'optimal'. - power_t : double + power_t : double, default=0.5 The exponent for inverse scaling learning rate [default 0.5]. early_stopping : bool, default=False @@ -861,7 +866,7 @@ class SGDClassifier(BaseSGDClassifier): .. versionadded:: 0.20 - class_weight : dict, {class_label: weight} or "balanced" or None, optional + class_weight : dict, {class_label: weight} or "balanced", default=None Preset for the class_weight fit parameter. Weights associated with classes. If not given, all classes @@ -893,11 +898,11 @@ class SGDClassifier(BaseSGDClassifier): Attributes ---------- - coef_ : array, shape (1, n_features) if n_classes == 2 else (n_classes,\ - n_features) + coef_ : ndarray of shape (1, n_features) if n_classes == 2 else \ + (n_classes, n_features) Weights assigned to the features. - intercept_ : array, shape (1,) if n_classes == 2 else (n_classes,) + intercept_ : ndarray of shape (1,) if n_classes == 2 else (n_classes,) Constants in decision function. n_iter_ : int @@ -979,7 +984,7 @@ def predict_proba(self): Returns ------- - array, shape (n_samples, n_classes) + ndarray of shape (n_samples, n_classes) Returns the probability of the sample for each class in the model, where classes are ordered as they are in `self.classes_`. @@ -1098,8 +1103,9 @@ def __init__(self, loss="squared_loss", penalty="l2", alpha=0.0001, def _partial_fit(self, X, y, alpha, C, loss, learning_rate, max_iter, sample_weight, coef_init, intercept_init): - X, y = check_X_y(X, y, "csr", copy=False, order='C', dtype=np.float64, - accept_large_sparse=False) + X, y = self._validate_data(X, y, accept_sparse="csr", copy=False, + order='C', dtype=np.float64, + accept_large_sparse=False) y = y.astype(np.float64, copy=False) n_samples, n_features = X.shape @@ -1113,11 +1119,11 @@ def _partial_fit(self, X, y, alpha, C, loss, learning_rate, elif n_features != self.coef_.shape[-1]: raise ValueError("Number of features %d does not match previous " "data %d." % (n_features, self.coef_.shape[-1])) - if self.average > 0 and getattr(self, "average_coef_", None) is None: - self.average_coef_ = np.zeros(n_features, + if self.average > 0 and getattr(self, "_average_coef", None) is None: + self._average_coef = np.zeros(n_features, dtype=np.float64, order="C") - self.average_intercept_ = np.zeros(1, dtype=np.float64, order="C") + self._average_intercept = np.zeros(1, dtype=np.float64, order="C") self._fit_regressor(X, y, alpha, C, loss, learning_rate, sample_weight, max_iter) @@ -1140,7 +1146,7 @@ def partial_fit(self, X, y, sample_weight=None): y : numpy array of shape (n_samples,) Subset of target values - sample_weight : array-like, shape (n_samples,), optional + sample_weight : array-like, shape (n_samples,), default=None Weights applied to individual samples. If not provided, uniform weights are assumed. @@ -1167,12 +1173,6 @@ def _fit(self, X, y, alpha, C, loss, learning_rate, coef_init=None, self.coef_ = None self.intercept_ = None - if self.average > 0: - self.standard_intercept_ = self.intercept_ - self.standard_coef_ = self.coef_ - self.average_coef_ = None - self.average_intercept_ = None - # Clear iteration count for multiple call to fit. self.t_ = 1.0 @@ -1198,16 +1198,16 @@ def fit(self, X, y, coef_init=None, intercept_init=None, X : {array-like, sparse matrix}, shape (n_samples, n_features) Training data - y : numpy array, shape (n_samples,) + y : ndarray of shape (n_samples,) Target values - coef_init : array, shape (n_features,) + coef_init : ndarray of shape (n_features,), default=None The initial coefficients to warm-start the optimization. - intercept_init : array, shape (1,) + intercept_init : ndarray of shape (1,), default=None The initial intercept to warm-start the optimization. - sample_weight : array-like, shape (n_samples,), optional + sample_weight : array-like, shape (n_samples,), default=None Weights applied to individual samples (1. for unweighted). Returns @@ -1229,7 +1229,7 @@ def _decision_function(self, X): Returns ------- - array, shape (n_samples,) + ndarray of shape (n_samples,) Predicted target values per element in X. """ check_is_fitted(self) @@ -1249,7 +1249,7 @@ def predict(self, X): Returns ------- - array, shape (n_samples,) + ndarray of shape (n_samples,) Predicted target values per element in X. """ return self._decision_function(X) @@ -1276,66 +1276,56 @@ def _fit_regressor(self, X, y, alpha, C, loss, learning_rate, tol = self.tol if self.tol is not None else -np.inf + if self.average: + coef = self._standard_coef + intercept = self._standard_intercept + average_coef = self._average_coef + average_intercept = self._average_intercept + else: + coef = self.coef_ + intercept = self.intercept_ + average_coef = None # Not used + average_intercept = [0] # Not used + + coef, intercept, average_coef, average_intercept, self.n_iter_ = \ + _plain_sgd(coef, + intercept[0], + average_coef, + average_intercept[0], + loss_function, + penalty_type, + alpha, C, + self.l1_ratio, + dataset, + validation_mask, self.early_stopping, + validation_score_cb, + int(self.n_iter_no_change), + max_iter, tol, + int(self.fit_intercept), + int(self.verbose), + int(self.shuffle), + seed, + 1.0, 1.0, + learning_rate_type, + self.eta0, self.power_t, self.t_, + intercept_decay, self.average) + + self.t_ += self.n_iter_ * X.shape[0] + if self.average > 0: - self.standard_coef_, self.standard_intercept_, \ - self.average_coef_, self.average_intercept_, self.n_iter_ =\ - average_sgd(self.standard_coef_, - self.standard_intercept_[0], - self.average_coef_, - self.average_intercept_[0], - loss_function, - penalty_type, - alpha, C, - self.l1_ratio, - dataset, - validation_mask, self.early_stopping, - validation_score_cb, - int(self.n_iter_no_change), - max_iter, tol, - int(self.fit_intercept), - int(self.verbose), - int(self.shuffle), - seed, - 1.0, 1.0, - learning_rate_type, - self.eta0, self.power_t, self.t_, - intercept_decay, self.average) - - self.average_intercept_ = np.atleast_1d(self.average_intercept_) - self.standard_intercept_ = np.atleast_1d(self.standard_intercept_) - self.t_ += self.n_iter_ * X.shape[0] + self._average_intercept = np.atleast_1d(average_intercept) + self._standard_intercept = np.atleast_1d(intercept) if self.average <= self.t_ - 1.0: - self.coef_ = self.average_coef_ - self.intercept_ = self.average_intercept_ + # made enough updates for averaging to be taken into account + self.coef_ = average_coef + self.intercept_ = np.atleast_1d(average_intercept) else: - self.coef_ = self.standard_coef_ - self.intercept_ = self.standard_intercept_ + self.coef_ = coef + self.intercept_ = np.atleast_1d(intercept) else: - self.coef_, self.intercept_, self.n_iter_ = \ - plain_sgd(self.coef_, - self.intercept_[0], - loss_function, - penalty_type, - alpha, C, - self.l1_ratio, - dataset, - validation_mask, self.early_stopping, - validation_score_cb, - int(self.n_iter_no_change), - max_iter, tol, - int(self.fit_intercept), - int(self.verbose), - int(self.shuffle), - seed, - 1.0, 1.0, - learning_rate_type, - self.eta0, self.power_t, self.t_, - intercept_decay) - - self.t_ += self.n_iter_ * X.shape[0] - self.intercept_ = np.atleast_1d(self.intercept_) + self.intercept_ = np.atleast_1d(intercept) class SGDRegressor(BaseSGDRegressor): @@ -1359,7 +1349,7 @@ class SGDRegressor(BaseSGDRegressor): Parameters ---------- - loss : str, default: 'squared_loss' + loss : str, default='squared_loss' The loss function to be used. The possible values are 'squared_loss', 'huber', 'epsilon_insensitive', or 'squared_epsilon_insensitive' @@ -1371,44 +1361,42 @@ class SGDRegressor(BaseSGDRegressor): 'squared_epsilon_insensitive' is the same but becomes squared loss past a tolerance of epsilon. - penalty : str, 'none', 'l2', 'l1', or 'elasticnet' + penalty : {'l2', 'l1', 'elasticnet'}, default='l2' The penalty (aka regularization term) to be used. Defaults to 'l2' which is the standard regularizer for linear SVM models. 'l1' and 'elasticnet' might bring sparsity to the model (feature selection) not achievable with 'l2'. - alpha : float - Constant that multiplies the regularization term. Defaults to 0.0001 + alpha : float, default=0.0001 + Constant that multiplies the regularization term. Also used to compute learning_rate when set to 'optimal'. - l1_ratio : float + l1_ratio : float, default=0.15 The Elastic Net mixing parameter, with 0 <= l1_ratio <= 1. l1_ratio=0 corresponds to L2 penalty, l1_ratio=1 to L1. - Defaults to 0.15. - fit_intercept : bool + fit_intercept : bool, default=True Whether the intercept should be estimated or not. If False, the - data is assumed to be already centered. Defaults to True. + data is assumed to be already centered. - max_iter : int, optional (default=1000) + max_iter : int, default=1000 The maximum number of passes over the training data (aka epochs). It only impacts the behavior in the ``fit`` method, and not the :meth:`partial_fit` method. .. versionadded:: 0.19 - tol : float or None, optional (default=1e-3) + tol : float, default=1e-3 The stopping criterion. If it is not None, the iterations will stop when (loss > best_loss - tol) for ``n_iter_no_change`` consecutive epochs. .. versionadded:: 0.19 - shuffle : bool, optional + shuffle : bool, default=True Whether or not the training data should be shuffled after each epoch. - Defaults to True. - verbose : integer, default=0 + verbose : int, default=0 The verbosity level. epsilon : float, default=0.1 @@ -1419,14 +1407,12 @@ class SGDRegressor(BaseSGDRegressor): For epsilon-insensitive, any differences between the current prediction and the correct label are ignored if they are less than this threshold. - random_state : int, RandomState instance or None, optional (default=None) - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. + random_state : int, RandomState instance, default=None + Used for shuffling the data, when ``shuffle`` is set to ``True``. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. - learning_rate : string, optional + learning_rate : string, default='invscaling' The learning rate schedule: 'constant': @@ -1442,12 +1428,12 @@ class SGDRegressor(BaseSGDRegressor): training loss by tol or fail to increase validation score by tol if early_stopping is True, the current learning rate is divided by 5. - eta0 : double + eta0 : double, default=0.01 The initial learning rate for the 'constant', 'invscaling' or 'adaptive' schedules. The default value is 0.01. - power_t : double - The exponent for inverse scaling learning rate [default 0.25]. + power_t : double, default=0.25 + The exponent for inverse scaling learning rate. early_stopping : bool, default=False Whether to use early stopping to terminate training when validation @@ -1492,17 +1478,26 @@ class SGDRegressor(BaseSGDRegressor): Attributes ---------- - coef_ : array, shape (n_features,) + coef_ : ndarray of shape (n_features,) Weights assigned to the features. - intercept_ : array, shape (1,) + intercept_ : ndarray of shape (1,) The intercept term. - average_coef_ : array, shape (n_features,) - Averaged weights assigned to the features. + average_coef_ : ndarray of shape (n_features,) + Averaged weights assigned to the features. Only available + if ``average=True``. + + .. deprecated:: 0.23 + Attribute ``average_coef_`` was deprecated + in version 0.23 and will be removed in 0.25. + + average_intercept_ : ndarray of shape (1,) + The averaged intercept term. Only available if ``average=True``. - average_intercept_ : array, shape (1,) - The averaged intercept term. + .. deprecated:: 0.23 + Attribute ``average_intercept_`` was deprecated + in version 0.23 and will be removed in 0.25. n_iter_ : int The actual number of iterations to reach the stopping criterion. diff --git a/sklearn/linear_model/_theil_sen.py b/sklearn/linear_model/_theil_sen.py index 9adf8109a10ef..a29cc26cdc0a3 100644 --- a/sklearn/linear_model/_theil_sen.py +++ b/sklearn/linear_model/_theil_sen.py @@ -240,12 +240,11 @@ class TheilSenRegressor(RegressorMixin, LinearModel): tol : float, optional, default 1.e-3 Tolerance when calculating spatial median. - random_state : int, RandomState instance or None, optional, default None + random_state : int, RandomState instance, default=None A random number generator instance to define the state of the random - permutations generator. If int, random_state is the seed used by the - random number generator; If RandomState instance, random_state is the - random number generator; If None, the random number generator is the - RandomState instance used by `np.random`. + permutations generator. Pass an int for reproducible output across + multiple function calls. + See :term:`Glossary ` n_jobs : int or None, optional (default=None) Number of CPUs to use during the cross validation. @@ -358,7 +357,7 @@ def fit(self, X, y): self : returns an instance of self. """ random_state = check_random_state(self.random_state) - X, y = check_X_y(X, y, y_numeric=True) + X, y = self._validate_data(X, y, y_numeric=True) n_samples, n_features = X.shape n_subsamples, self.n_subpopulation_ = self._check_subparams(n_samples, n_features) diff --git a/sklearn/linear_model/setup.py b/sklearn/linear_model/setup.py index 121b449d673d0..d0c9e8c04c16d 100644 --- a/sklearn/linear_model/setup.py +++ b/sklearn/linear_model/setup.py @@ -33,6 +33,8 @@ def configuration(parent_package='', top_path=None): # add other directories config.add_subpackage('tests') + config.add_subpackage('_glm') + config.add_subpackage('_glm/tests') return config diff --git a/sklearn/linear_model/tests/test_base.py b/sklearn/linear_model/tests/test_base.py index a932d5ed33fe1..c962edccc953a 100644 --- a/sklearn/linear_model/tests/test_base.py +++ b/sklearn/linear_model/tests/test_base.py @@ -5,6 +5,8 @@ import pytest +from distutils.version import LooseVersion + import numpy as np from scipy import sparse from scipy import linalg @@ -129,11 +131,11 @@ def test_fit_intercept(): lr3_with_intercept = LinearRegression().fit(X3, y) assert (lr2_with_intercept.coef_.shape == - lr2_without_intercept.coef_.shape) + lr2_without_intercept.coef_.shape) assert (lr3_with_intercept.coef_.shape == - lr3_without_intercept.coef_.shape) + lr3_without_intercept.coef_.shape) assert (lr2_without_intercept.coef_.ndim == - lr3_without_intercept.coef_.ndim) + lr3_without_intercept.coef_.ndim) def test_linear_regression_sparse(random_state=0): @@ -205,6 +207,22 @@ def test_linear_regression_sparse_multiple_outcome(random_state=0): assert_array_almost_equal(np.vstack((y_pred, y_pred)).T, Y_pred, decimal=3) +def test_linear_regression_pd_sparse_dataframe_warning(): + pd = pytest.importorskip('pandas') + # restrict the pd versions < '0.24.0' as they have a bug in is_sparse func + if LooseVersion(pd.__version__) < '0.24.0': + pytest.skip("pandas 0.24+ required.") + df = pd.DataFrame() + for col in range(4): + arr = np.random.randn(10) + arr[:8] = 0 + df[str(col)] = pd.arrays.SparseArray(arr, fill_value=0) + msg = "pandas.DataFrame with sparse columns found." + with pytest.warns(UserWarning, match=msg): + reg = LinearRegression() + reg.fit(df.iloc[:, 0:2], df.iloc[:, 3]) + + def test_preprocess_data(): n_samples = 200 n_features = 2 @@ -433,16 +451,23 @@ def test_dtype_preprocess_data(): assert_array_almost_equal(X_norm_32, X_norm_64) -def test_rescale_data(): +@pytest.mark.parametrize('n_targets', [None, 2]) +def test_rescale_data_dense(n_targets): n_samples = 200 n_features = 2 sample_weight = 1.0 + rng.rand(n_samples) X = rng.rand(n_samples, n_features) - y = rng.rand(n_samples) + if n_targets is None: + y = rng.rand(n_samples) + else: + y = rng.rand(n_samples, n_targets) rescaled_X, rescaled_y = _rescale_data(X, y, sample_weight) rescaled_X2 = X * np.sqrt(sample_weight)[:, np.newaxis] - rescaled_y2 = y * np.sqrt(sample_weight) + if n_targets is None: + rescaled_y2 = y * np.sqrt(sample_weight) + else: + rescaled_y2 = y * np.sqrt(sample_weight)[:, np.newaxis] assert_array_almost_equal(rescaled_X, rescaled_X2) assert_array_almost_equal(rescaled_y, rescaled_y2) diff --git a/sklearn/linear_model/tests/test_bayes.py b/sklearn/linear_model/tests/test_bayes.py index 9cbd8f9970d9d..e1922a010514f 100644 --- a/sklearn/linear_model/tests/test_bayes.py +++ b/sklearn/linear_model/tests/test_bayes.py @@ -209,7 +209,7 @@ def test_ard_accuracy_on_easy_problem(): X = np.random.RandomState(seed=seed).normal(size=(250, 3)) y = X[:, 1] - regressor = ARDRegression() + regressor = ARDRegression(n_iter=600) regressor.fit(X, y) abs_coef_error = np.abs(1 - regressor.coef_[1]) diff --git a/sklearn/linear_model/tests/test_coordinate_descent.py b/sklearn/linear_model/tests/test_coordinate_descent.py index a739c876fa77f..fdc49599788fe 100644 --- a/sklearn/linear_model/tests/test_coordinate_descent.py +++ b/sklearn/linear_model/tests/test_coordinate_descent.py @@ -3,6 +3,7 @@ # License: BSD 3 clause import numpy as np +from numpy.testing import assert_allclose import pytest from scipy import interpolate, sparse from copy import deepcopy @@ -24,9 +25,47 @@ LassoCV, ElasticNet, ElasticNetCV, MultiTaskLasso, MultiTaskElasticNet, \ MultiTaskElasticNetCV, MultiTaskLassoCV, lasso_path, enet_path from sklearn.linear_model import LassoLarsCV, lars_path +from sklearn.linear_model._coordinate_descent import _set_order from sklearn.utils import check_array +@pytest.mark.parametrize('order', ['C', 'F']) +@pytest.mark.parametrize('input_order', ['C', 'F']) +def test_set_order_dense(order, input_order): + """Check that _set_order returns arrays with promised order.""" + X = np.array([[0], [0], [0]], order=input_order) + y = np.array([0, 0, 0], order=input_order) + X2, y2 = _set_order(X, y, order=order) + if order == 'C': + assert X2.flags['C_CONTIGUOUS'] + assert y2.flags['C_CONTIGUOUS'] + elif order == 'F': + assert X2.flags['F_CONTIGUOUS'] + assert y2.flags['F_CONTIGUOUS'] + + if order == input_order: + assert X is X2 + assert y is y2 + + +@pytest.mark.parametrize('order', ['C', 'F']) +@pytest.mark.parametrize('input_order', ['C', 'F']) +def test_set_order_sparse(order, input_order): + """Check that _set_order returns sparse matrices in promised format.""" + X = sparse.coo_matrix(np.array([[0], [0], [0]])) + y = sparse.coo_matrix(np.array([0, 0, 0])) + sparse_format = "csc" if input_order == "F" else "csr" + X = X.asformat(sparse_format) + y = X.asformat(sparse_format) + X2, y2 = _set_order(X, y, order=order) + if order == 'C': + assert sparse.isspmatrix_csr(X2) + assert sparse.isspmatrix_csr(y2) + elif order == 'F': + assert sparse.isspmatrix_csc(X2) + assert sparse.isspmatrix_csc(y2) + + def test_lasso_zero(): # Check that the lasso can handle zero data without crashing X = [[0], [0], [0]] @@ -173,7 +212,7 @@ def test_lasso_cv(): def test_lasso_cv_with_some_model_selection(): from sklearn.pipeline import make_pipeline from sklearn.preprocessing import StandardScaler - from sklearn.model_selection import StratifiedKFold + from sklearn.model_selection import ShuffleSplit from sklearn import datasets from sklearn.linear_model import LassoCV @@ -183,7 +222,7 @@ def test_lasso_cv_with_some_model_selection(): pipe = make_pipeline( StandardScaler(), - LassoCV(cv=StratifiedKFold()) + LassoCV(cv=ShuffleSplit(random_state=0)) ) pipe.fit(X, y) @@ -229,7 +268,6 @@ def test_lasso_path_return_models_vs_new_return_gives_same_coefficients(): decimal=1) -@pytest.mark.filterwarnings('ignore: The default value of multioutput') # 0.23 def test_enet_path(): # We use a large number of samples and of informative features so that # the l1_ratio selected is more toward ridge than lasso @@ -898,3 +936,87 @@ def test_multi_task_lasso_cv_dtype(): y = X[:, [0, 0]].copy() est = MultiTaskLassoCV(n_alphas=5, fit_intercept=True).fit(X, y) assert_array_almost_equal(est.coef_, [[1, 0, 0]] * 2, decimal=3) + + +@pytest.mark.parametrize('fit_intercept', [True, False]) +@pytest.mark.parametrize('alpha', [0.01]) +@pytest.mark.parametrize('normalize', [False, True]) +@pytest.mark.parametrize('precompute', [False, True]) +def test_enet_sample_weight_consistency(fit_intercept, alpha, normalize, + precompute): + """Test that the impact of sample_weight is consistent.""" + rng = np.random.RandomState(0) + n_samples, n_features = 10, 5 + + X = rng.rand(n_samples, n_features) + y = rng.rand(n_samples) + params = dict(alpha=alpha, fit_intercept=fit_intercept, + precompute=precompute, tol=1e-6, l1_ratio=0.5) + + reg = ElasticNet(**params).fit(X, y) + coef = reg.coef_.copy() + if fit_intercept: + intercept = reg.intercept_ + + # sample_weight=np.ones(..) should be equivalent to sample_weight=None + sample_weight = np.ones_like(y) + reg.fit(X, y, sample_weight=sample_weight) + assert_allclose(reg.coef_, coef, rtol=1e-6) + if fit_intercept: + assert_allclose(reg.intercept_, intercept) + + # sample_weight=None should be equivalent to sample_weight = number + sample_weight = 123. + reg.fit(X, y, sample_weight=sample_weight) + assert_allclose(reg.coef_, coef, rtol=1e-6) + if fit_intercept: + assert_allclose(reg.intercept_, intercept) + + # scaling of sample_weight should have no effect, cf. np.average() + sample_weight = 2 * np.ones_like(y) + reg.fit(X, y, sample_weight=sample_weight) + assert_allclose(reg.coef_, coef, rtol=1e-6) + if fit_intercept: + assert_allclose(reg.intercept_, intercept) + + # setting one element of sample_weight to 0 is equivalent to removing + # the corresponding sample + sample_weight = np.ones_like(y) + sample_weight[-1] = 0 + reg.fit(X, y, sample_weight=sample_weight) + coef1 = reg.coef_.copy() + if fit_intercept: + intercept1 = reg.intercept_ + reg.fit(X[:-1], y[:-1]) + assert_allclose(reg.coef_, coef1, rtol=1e-6) + if fit_intercept: + assert_allclose(reg.intercept_, intercept1) + + # check that multiplying sample_weight by 2 is equivalent + # to repeating corresponding samples twice + if sparse.issparse(X): + X = X.toarray() + + X2 = np.concatenate([X, X[:n_samples//2]], axis=0) + y2 = np.concatenate([y, y[:n_samples//2]]) + sample_weight_1 = np.ones(len(y)) + sample_weight_1[:n_samples//2] = 2 + + reg1 = ElasticNet(**params).fit( + X, y, sample_weight=sample_weight_1 + ) + + reg2 = ElasticNet(**params).fit( + X2, y2, sample_weight=None + ) + assert_allclose(reg1.coef_, reg2.coef_) + + +def test_enet_sample_weight_sparse(): + reg = ElasticNet() + X = sparse.csc_matrix(np.zeros((3, 2))) + y = np.array([-1, 0, 1]) + sw = np.array([1, 2, 3]) + with pytest.raises(ValueError, match="Sample weights do not.*support " + "sparse matrices"): + reg.fit(X, y, sample_weight=sw, check_input=True) diff --git a/sklearn/linear_model/tests/test_huber.py b/sklearn/linear_model/tests/test_huber.py index 78fa0f3b1cd14..cb70db88d3d41 100644 --- a/sklearn/linear_model/tests/test_huber.py +++ b/sklearn/linear_model/tests/test_huber.py @@ -143,8 +143,6 @@ def test_huber_scaling_invariant(): assert_array_equal(n_outliers_mask_3, n_outliers_mask_1) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_huber_and_sgd_same_results(): # Test they should converge to same coefficients for same parameters diff --git a/sklearn/linear_model/tests/test_least_angle.py b/sklearn/linear_model/tests/test_least_angle.py index 2b7ed5a83b8d8..6e7c1fb37096a 100644 --- a/sklearn/linear_model/tests/test_least_angle.py +++ b/sklearn/linear_model/tests/test_least_angle.py @@ -15,7 +15,8 @@ from sklearn.utils._testing import TempMemmap from sklearn.exceptions import ConvergenceWarning from sklearn import linear_model, datasets -from sklearn.linear_model._least_angle import _lars_path_residues, LassoLarsIC +from sklearn.linear_model._least_angle import _lars_path_residues +from sklearn.linear_model import LassoLarsIC, lars_path # TODO: use another dataset that has multiple drops diabetes = datasets.load_diabetes() @@ -730,3 +731,9 @@ def test_lasso_lars_fit_copyX_behaviour(copy_X): y = X[:, 2] lasso_lars.fit(X, y, copy_X=copy_X) assert copy_X == np.array_equal(X, X_copy) + + +def test_X_none_gram_not_none(): + with pytest.raises(ValueError, + match="X cannot be None if Gram is not None"): + lars_path(X=None, y=[1], Gram='not None') diff --git a/sklearn/linear_model/tests/test_logistic.py b/sklearn/linear_model/tests/test_logistic.py index 894040c2053bd..3590793de5071 100644 --- a/sklearn/linear_model/tests/test_logistic.py +++ b/sklearn/linear_model/tests/test_logistic.py @@ -31,10 +31,8 @@ from sklearn.utils._testing import skip_if_no_parallel from sklearn.exceptions import ConvergenceWarning -from sklearn.exceptions import ChangedBehaviorWarning from sklearn.linear_model._logistic import ( LogisticRegression, - logistic_regression_path, _logistic_regression_path, LogisticRegressionCV, _logistic_loss_and_grad, _logistic_grad_hess, _multinomial_grad_hess, _logistic_loss, @@ -390,8 +388,20 @@ def test_logistic_regression_path_convergence_fail(): X = np.concatenate((rng.randn(100, 2) + [1, 1], rng.randn(100, 2))) y = [1] * 100 + [-1] * 100 Cs = [1e3] - assert_warns(ConvergenceWarning, _logistic_regression_path, - X, y, Cs=Cs, tol=0., max_iter=1, random_state=0, verbose=1) + + # Check that the convergence message points to both a model agnostic + # advice (scaling the data) and to the logistic regression specific + # documentation that includes hints on the solver configuration. + with pytest.warns(ConvergenceWarning) as record: + _logistic_regression_path( + X, y, Cs=Cs, tol=0., max_iter=1, random_state=0, verbose=0) + + assert len(record) == 1 + warn_msg = record[0].message.args[0] + assert "lbfgs failed to converge" in warn_msg + assert "Increase the number of iterations" in warn_msg + assert "scale the data" in warn_msg + assert "linear_model.html#logistic-regression" in warn_msg def test_liblinear_dual_random_state(): @@ -1713,7 +1723,7 @@ def fit(X, y, **kw): if sys.platform == 'darwin' and solver == 'lbfgs': pytest.xfail('Issue #11924: LogisticRegressionCV(solver="lbfgs", ' 'multi_class="multinomial") is nondterministic on ' - 'MacOS.') # pragma: no cover + 'MacOS.') assert_allclose(est_auto_multi.coef_, est_multi_multi.coef_) assert_allclose(est_auto_multi.predict_proba(X2), est_multi_multi.predict_proba(X2)) @@ -1727,13 +1737,6 @@ def fit(X, y, **kw): solver=solver).coef_) -def test_logistic_regression_path_deprecation(): - - assert_warns_message(FutureWarning, - "logistic_regression_path was deprecated", - logistic_regression_path, X, Y1) - - @pytest.mark.parametrize('solver', ('lbfgs', 'newton-cg', 'sag', 'saga')) def test_penalty_none(solver): # - Make sure warning is raised if penalty='none' and C is set to a diff --git a/sklearn/linear_model/tests/test_passive_aggressive.py b/sklearn/linear_model/tests/test_passive_aggressive.py index 5da9883cba369..27381059eaf33 100644 --- a/sklearn/linear_model/tests/test_passive_aggressive.py +++ b/sklearn/linear_model/tests/test_passive_aggressive.py @@ -67,8 +67,6 @@ def project(self, X): return np.dot(X, self.w) + self.b -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_classifier_accuracy(): for data in (X, X_csr): for fit_intercept in (True, False): @@ -80,33 +78,30 @@ def test_classifier_accuracy(): score = clf.score(data, y) assert score > 0.79 if average: - assert hasattr(clf, 'average_coef_') - assert hasattr(clf, 'average_intercept_') - assert hasattr(clf, 'standard_intercept_') - assert hasattr(clf, 'standard_coef_') + assert hasattr(clf, '_average_coef') + assert hasattr(clf, '_average_intercept') + assert hasattr(clf, '_standard_intercept') + assert hasattr(clf, '_standard_coef') -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_classifier_partial_fit(): classes = np.unique(y) for data in (X, X_csr): for average in (False, True): clf = PassiveAggressiveClassifier(random_state=0, - average=average, max_iter=5) + average=average, + max_iter=5) for t in range(30): clf.partial_fit(data, y, classes) score = clf.score(data, y) assert score > 0.79 if average: - assert hasattr(clf, 'average_coef_') - assert hasattr(clf, 'average_intercept_') - assert hasattr(clf, 'standard_intercept_') - assert hasattr(clf, 'standard_coef_') + assert hasattr(clf, '_average_coef') + assert hasattr(clf, '_average_intercept') + assert hasattr(clf, '_standard_intercept') + assert hasattr(clf, '_standard_coef') -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_classifier_refit(): # Classifier can be retrained on different labels and features. clf = PassiveAggressiveClassifier(max_iter=5).fit(X, y) @@ -116,8 +111,6 @@ def test_classifier_refit(): assert_array_equal(clf.classes_, iris.target_names) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') @pytest.mark.parametrize('loss', ("hinge", "squared_hinge")) def test_classifier_correctness(loss): y_bin = y.copy() @@ -128,7 +121,7 @@ def test_classifier_correctness(loss): for data in (X, X_csr): clf2 = PassiveAggressiveClassifier(loss=loss, max_iter=2, - shuffle=False, tol=None) + shuffle=False, tol=None) clf2.fit(data, y_bin) assert_array_almost_equal(clf1.w, clf2.coef_.ravel(), decimal=2) @@ -140,8 +133,6 @@ def test_classifier_undefined_methods(): assert_raises(AttributeError, lambda x: getattr(clf, x), meth) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_class_weights(): # Test class weights. X2 = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0], @@ -164,16 +155,12 @@ def test_class_weights(): assert_array_equal(clf.predict([[0.2, -1.0]]), np.array([-1])) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_partial_fit_weight_class_balanced(): # partial_fit with class_weight='balanced' not supported clf = PassiveAggressiveClassifier(class_weight="balanced", max_iter=100) assert_raises(ValueError, clf.partial_fit, X, y, classes=np.unique(y)) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_equal_class_weight(): X2 = [[1, 0], [1, 0], [0, 1], [0, 1]] y2 = [0, 0, 1, 1] @@ -195,8 +182,6 @@ def test_equal_class_weight(): assert_almost_equal(clf.coef_, clf_balanced.coef_, decimal=2) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_wrong_class_weight_label(): # ValueError due to wrong class_weight label. X2 = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0], @@ -207,8 +192,6 @@ def test_wrong_class_weight_label(): assert_raises(ValueError, clf.fit, X2, y2) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_wrong_class_weight_format(): # ValueError due to wrong class_weight argument type. X2 = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0], @@ -222,8 +205,6 @@ def test_wrong_class_weight_format(): assert_raises(ValueError, clf.fit, X2, y2) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_regressor_mse(): y_bin = y.copy() y_bin[y != 1] = -1 @@ -238,14 +219,12 @@ def test_regressor_mse(): pred = reg.predict(data) assert np.mean((pred - y_bin) ** 2) < 1.7 if average: - assert hasattr(reg, 'average_coef_') - assert hasattr(reg, 'average_intercept_') - assert hasattr(reg, 'standard_intercept_') - assert hasattr(reg, 'standard_coef_') + assert hasattr(reg, '_average_coef') + assert hasattr(reg, '_average_intercept') + assert hasattr(reg, '_standard_intercept') + assert hasattr(reg, '_standard_coef') -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_regressor_partial_fit(): y_bin = y.copy() y_bin[y != 1] = -1 @@ -253,20 +232,18 @@ def test_regressor_partial_fit(): for data in (X, X_csr): for average in (False, True): reg = PassiveAggressiveRegressor(random_state=0, - average=average, max_iter=100) + average=average, max_iter=100) for t in range(50): reg.partial_fit(data, y_bin) pred = reg.predict(data) assert np.mean((pred - y_bin) ** 2) < 1.7 if average: - assert hasattr(reg, 'average_coef_') - assert hasattr(reg, 'average_intercept_') - assert hasattr(reg, 'standard_intercept_') - assert hasattr(reg, 'standard_coef_') + assert hasattr(reg, '_average_coef') + assert hasattr(reg, '_average_intercept') + assert hasattr(reg, '_standard_intercept') + assert hasattr(reg, '_standard_coef') -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') @pytest.mark.parametrize( 'loss', ("epsilon_insensitive", "squared_epsilon_insensitive")) @@ -279,7 +256,7 @@ def test_regressor_correctness(loss): for data in (X, X_csr): reg2 = PassiveAggressiveRegressor(tol=None, loss=loss, max_iter=2, - shuffle=False) + shuffle=False) reg2.fit(data, y_bin) assert_array_almost_equal(reg1.w, reg2.coef_.ravel(), decimal=2) @@ -289,3 +266,16 @@ def test_regressor_undefined_methods(): reg = PassiveAggressiveRegressor(max_iter=100) for meth in ("transform",): assert_raises(AttributeError, lambda x: getattr(reg, x), meth) + +# TODO: remove in 0.25 +@pytest.mark.parametrize('klass', [PassiveAggressiveClassifier, + PassiveAggressiveRegressor]) +def test_passive_aggressive_deprecated_attr(klass): + est = klass(average=True) + est.fit(X, y) + + msg = "Attribute {} was deprecated" + for att in ['average_coef_', 'average_intercept_', + 'standard_coef_', 'standard_intercept_']: + with pytest.warns(FutureWarning, match=msg.format(att)): + getattr(est, att) diff --git a/sklearn/linear_model/tests/test_perceptron.py b/sklearn/linear_model/tests/test_perceptron.py index ffbd844b902f2..6cdd538ca9247 100644 --- a/sklearn/linear_model/tests/test_perceptron.py +++ b/sklearn/linear_model/tests/test_perceptron.py @@ -43,8 +43,6 @@ def predict(self, X): return np.sign(self.project(X)) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_perceptron_accuracy(): for data in (X, X_csr): clf = Perceptron(max_iter=100, tol=None, shuffle=False) @@ -53,8 +51,6 @@ def test_perceptron_accuracy(): assert score > 0.7 -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_perceptron_correctness(): y_bin = y.copy() y_bin[y != 1] = -1 diff --git a/sklearn/linear_model/tests/test_ransac.py b/sklearn/linear_model/tests/test_ransac.py index 83f688c95692e..f52e4f0852d5f 100644 --- a/sklearn/linear_model/tests/test_ransac.py +++ b/sklearn/linear_model/tests/test_ransac.py @@ -7,10 +7,12 @@ from sklearn.utils import check_random_state from sklearn.utils._testing import assert_warns -from sklearn.utils._testing import assert_almost_equal from sklearn.utils._testing import assert_raises_regexp from sklearn.utils._testing import assert_raises -from sklearn.linear_model import LinearRegression, RANSACRegressor, Lasso +from sklearn.utils._testing import assert_allclose +from sklearn.datasets import make_regression +from sklearn.linear_model import LinearRegression, RANSACRegressor +from sklearn.linear_model import OrthogonalMatchingPursuit from sklearn.linear_model._ransac import _dynamic_max_trials from sklearn.exceptions import ConvergenceWarning @@ -332,7 +334,6 @@ def test_ransac_min_n_samples(): assert_raises(ValueError, ransac_estimator7.fit, X, y) -@pytest.mark.filterwarnings('ignore: The default value of multioutput') # 0.23 def test_ransac_multi_dimensional_targets(): base_estimator = LinearRegression() @@ -353,7 +354,6 @@ def test_ransac_multi_dimensional_targets(): assert_array_equal(ransac_estimator.inlier_mask_, ref_inlier_mask) -@pytest.mark.filterwarnings('ignore: The default value of multioutput') # 0.23 def test_ransac_residual_loss(): loss_multi1 = lambda y_true, y_pred: np.sum(np.abs(y_true - y_pred), axis=1) loss_multi2 = lambda y_true, y_pred: np.sum((y_true - y_pred) ** 2, axis=1) @@ -487,10 +487,28 @@ def test_ransac_fit_sample_weight(): y_ = np.append(y_, outlier_y) ransac_estimator.fit(X_, y_, sample_weight) - assert_almost_equal(ransac_estimator.estimator_.coef_, ref_coef_) + assert_allclose(ransac_estimator.estimator_.coef_, ref_coef_) # check that if base_estimator.fit doesn't support # sample_weight, raises error - base_estimator = Lasso() + base_estimator = OrthogonalMatchingPursuit() ransac_estimator = RANSACRegressor(base_estimator) assert_raises(ValueError, ransac_estimator.fit, X, y, weights) + + +def test_ransac_final_model_fit_sample_weight(): + X, y = make_regression(n_samples=1000, random_state=10) + rng = check_random_state(42) + sample_weight = rng.randint(1, 4, size=y.shape[0]) + sample_weight = sample_weight / sample_weight.sum() + ransac = RANSACRegressor(base_estimator=LinearRegression(), random_state=0) + ransac.fit(X, y, sample_weight=sample_weight) + + final_model = LinearRegression() + mask_samples = ransac.inlier_mask_ + final_model.fit( + X[mask_samples], y[mask_samples], + sample_weight=sample_weight[mask_samples] + ) + + assert_allclose(ransac.estimator_.coef_, final_model.coef_) diff --git a/sklearn/linear_model/tests/test_ridge.py b/sklearn/linear_model/tests/test_ridge.py index c786b154fcb85..c1f7bb86a7fcf 100644 --- a/sklearn/linear_model/tests/test_ridge.py +++ b/sklearn/linear_model/tests/test_ridge.py @@ -34,6 +34,7 @@ from sklearn.linear_model._ridge import _check_gcv_mode from sklearn.linear_model._ridge import _X_CenterStackOp from sklearn.datasets import make_regression +from sklearn.datasets import make_classification from sklearn.model_selection import GridSearchCV from sklearn.model_selection import KFold, GroupKFold, cross_val_predict @@ -59,6 +60,14 @@ SPARSE_FILTER = lambda X: sp.csr_matrix(X) +def _accuracy_callable(y_test, y_pred): + return np.mean(y_test == y_pred) + + +def _mean_squared_error_callable(y_test, y_pred): + return ((y_test - y_pred) ** 2).mean() + + @pytest.mark.parametrize('solver', ("svd", "sparse_cg", "cholesky", "lsqr", "sag")) def test_ridge(solver): @@ -661,6 +670,33 @@ def _test_ridge_cv(filter_): assert type(ridge_cv.intercept_) == np.float64 +@pytest.mark.parametrize( + "ridge, make_dataset", + [(RidgeCV(store_cv_values=False), make_regression), + (RidgeClassifierCV(store_cv_values=False), make_classification)] +) +def test_ridge_gcv_cv_values_not_stored(ridge, make_dataset): + # Check that `cv_values_` is not stored when store_cv_values is False + X, y = make_dataset(n_samples=6, random_state=42) + ridge.fit(X, y) + assert not hasattr(ridge, "cv_values_") + + +@pytest.mark.parametrize( + "ridge, make_dataset", + [(RidgeCV(), make_regression), + (RidgeClassifierCV(), make_classification)] +) +@pytest.mark.parametrize("cv", [None, 3]) +def test_ridge_best_score(ridge, make_dataset, cv): + # check that the best_score_ is store + X, y = make_dataset(n_samples=6, random_state=42) + ridge.set_params(store_cv_values=False, cv=cv) + ridge.fit(X, y) + assert hasattr(ridge, "best_score_") + assert isinstance(ridge.best_score_, float) + + def _test_ridge_diabetes(filter_): ridge = Ridge(fit_intercept=False) ridge.fit(filter_(X_diabetes), y_diabetes) @@ -698,6 +734,38 @@ def _test_ridge_classifiers(filter_): assert np.mean(y_iris == y_pred) >= 0.8 +@pytest.mark.parametrize("scoring", [None, "accuracy", _accuracy_callable]) +@pytest.mark.parametrize("cv", [None, KFold(5)]) +@pytest.mark.parametrize("filter_", [DENSE_FILTER, SPARSE_FILTER]) +def test_ridge_classifier_with_scoring(filter_, scoring, cv): + # non-regression test for #14672 + # check that RidgeClassifierCV works with all sort of scoring and + # cross-validation + scoring_ = make_scorer(scoring) if callable(scoring) else scoring + clf = RidgeClassifierCV(scoring=scoring_, cv=cv) + # Smoke test to check that fit/predict does not raise error + clf.fit(filter_(X_iris), y_iris).predict(filter_(X_iris)) + + +@pytest.mark.parametrize("cv", [None, KFold(5)]) +@pytest.mark.parametrize("filter_", [DENSE_FILTER, SPARSE_FILTER]) +def test_ridge_regression_custom_scoring(filter_, cv): + # check that custom scoring is working as expected + # check the tie breaking strategy (keep the first alpha tried) + + def _dummy_score(y_test, y_pred): + return 0.42 + + alphas = np.logspace(-2, 2, num=5) + clf = RidgeClassifierCV( + alphas=alphas, scoring=make_scorer(_dummy_score), cv=cv + ) + clf.fit(filter_(X_iris), y_iris) + assert clf.best_score_ == pytest.approx(0.42) + # In case of tie score, the first alphas will be kept + assert clf.alpha_ == pytest.approx(alphas[0]) + + def _test_tolerance(filter_): ridge = Ridge(tol=1e-5, fit_intercept=False) ridge.fit(filter_(X_diabetes), y_diabetes) @@ -720,7 +788,6 @@ def check_dense_sparse(test_func): assert_array_almost_equal(ret_dense, ret_sparse, decimal=3) -@pytest.mark.filterwarnings('ignore: The default value of multioutput') # 0.23 @pytest.mark.parametrize( 'test_func', (_test_ridge_loo, _test_ridge_cv, _test_ridge_cv_normalize, @@ -818,7 +885,10 @@ def test_class_weights_cv(): assert_array_equal(reg.predict([[-.2, 2]]), np.array([-1])) -def test_ridgecv_store_cv_values(): +@pytest.mark.parametrize( + "scoring", [None, 'neg_mean_squared_error', _mean_squared_error_callable] +) +def test_ridgecv_store_cv_values(scoring): rng = np.random.RandomState(42) n_samples = 8 @@ -827,7 +897,9 @@ def test_ridgecv_store_cv_values(): alphas = [1e-1, 1e0, 1e1] n_alphas = len(alphas) - r = RidgeCV(alphas=alphas, cv=None, store_cv_values=True) + scoring_ = make_scorer(scoring) if callable(scoring) else scoring + + r = RidgeCV(alphas=alphas, cv=None, store_cv_values=True, scoring=scoring_) # with len(y.shape) == 1 y = rng.randn(n_samples) @@ -840,12 +912,13 @@ def test_ridgecv_store_cv_values(): r.fit(x, y) assert r.cv_values_.shape == (n_samples, n_targets, n_alphas) - r = RidgeCV(cv=3, store_cv_values=True) + r = RidgeCV(cv=3, store_cv_values=True, scoring=scoring) assert_raises_regex(ValueError, 'cv!=None and store_cv_values', r.fit, x, y) -def test_ridge_classifier_cv_store_cv_values(): +@pytest.mark.parametrize("scoring", [None, 'accuracy', _accuracy_callable]) +def test_ridge_classifier_cv_store_cv_values(scoring): x = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0], [1.0, 1.0], [1.0, 0.0]]) y = np.array([1, 1, 1, -1, -1]) @@ -854,7 +927,11 @@ def test_ridge_classifier_cv_store_cv_values(): alphas = [1e-1, 1e0, 1e1] n_alphas = len(alphas) - r = RidgeClassifierCV(alphas=alphas, cv=None, store_cv_values=True) + scoring_ = make_scorer(scoring) if callable(scoring) else scoring + + r = RidgeClassifierCV( + alphas=alphas, cv=None, store_cv_values=True, scoring=scoring_ + ) # with len(y.shape) == 1 n_targets = 1 diff --git a/sklearn/linear_model/tests/test_sag.py b/sklearn/linear_model/tests/test_sag.py index 6a591288b55d8..6bb156c64715b 100644 --- a/sklearn/linear_model/tests/test_sag.py +++ b/sklearn/linear_model/tests/test_sag.py @@ -120,7 +120,7 @@ def sag(X, y, step_size, alpha, n_iter=1, dloss=None, sparse=False, def sag_sparse(X, y, step_size, alpha, n_iter=1, dloss=None, sample_weight=None, sparse=False, - fit_intercept=True, saga=False): + fit_intercept=True, saga=False, random_state=0): if step_size * alpha == 1.: raise ZeroDivisionError("Sparse sag does not handle the case " "step_size * alpha == 1") @@ -130,7 +130,7 @@ def sag_sparse(X, y, step_size, alpha, n_iter=1, sum_gradient = np.zeros(n_features) last_updated = np.zeros(n_features, dtype=np.int) gradient_memory = np.zeros(n_samples) - rng = np.random.RandomState(77) + rng = check_random_state(random_state) intercept = 0.0 intercept_sum_gradient = 0.0 wscale = 1.0 @@ -368,7 +368,7 @@ def test_sag_regressor_computed_correctly(): alpha = .1 n_features = 10 n_samples = 40 - max_iter = 50 + max_iter = 100 tol = .000001 fit_intercept = True rng = np.random.RandomState(0) @@ -378,7 +378,8 @@ def test_sag_regressor_computed_correctly(): step_size = get_step_size(X, alpha, fit_intercept, classification=False) clf1 = Ridge(fit_intercept=fit_intercept, tol=tol, solver='sag', - alpha=alpha * n_samples, max_iter=max_iter) + alpha=alpha * n_samples, max_iter=max_iter, + random_state=rng) clf2 = clone(clf1) clf1.fit(X, y) @@ -387,12 +388,14 @@ def test_sag_regressor_computed_correctly(): spweights1, spintercept1 = sag_sparse(X, y, step_size, alpha, n_iter=max_iter, dloss=squared_dloss, - fit_intercept=fit_intercept) + fit_intercept=fit_intercept, + random_state=rng) spweights2, spintercept2 = sag_sparse(X, y, step_size, alpha, n_iter=max_iter, dloss=squared_dloss, sparse=True, - fit_intercept=fit_intercept) + fit_intercept=fit_intercept, + random_state=rng) assert_array_almost_equal(clf1.coef_.ravel(), spweights1.ravel(), diff --git a/sklearn/linear_model/tests/test_sgd.py b/sklearn/linear_model/tests/test_sgd.py index f462a1fb4a040..22744a427b901 100644 --- a/sklearn/linear_model/tests/test_sgd.py +++ b/sklearn/linear_model/tests/test_sgd.py @@ -24,11 +24,6 @@ from sklearn.model_selection import RandomizedSearchCV -# 0.23. warning about tol not having its correct default value. -pytestmark = pytest.mark.filterwarnings( - "ignore:max_iter and tol parameters have been") - - def _update_kwargs(kwargs): if "random_state" not in kwargs: kwargs["random_state"] = 42 @@ -67,6 +62,7 @@ def partial_fit(self, X, y, *args, **kw): return linear_model.SGDRegressor.partial_fit(self, X, y, *args, **kw) def decision_function(self, X, *args, **kw): + # XXX untested as of v0.22 X = sp.csr_matrix(X) return linear_model.SGDRegressor.decision_function(self, X, *args, **kw) @@ -259,18 +255,31 @@ def test_plain_has_no_average_attr(klass): clf = klass(average=True, eta0=.01) clf.fit(X, Y) - assert hasattr(clf, 'average_coef_') - assert hasattr(clf, 'average_intercept_') - assert hasattr(clf, 'standard_intercept_') - assert hasattr(clf, 'standard_coef_') + assert hasattr(clf, '_average_coef') + assert hasattr(clf, '_average_intercept') + assert hasattr(clf, '_standard_intercept') + assert hasattr(clf, '_standard_coef') clf = klass() clf.fit(X, Y) - assert not hasattr(clf, 'average_coef_') - assert not hasattr(clf, 'average_intercept_') - assert not hasattr(clf, 'standard_intercept_') - assert not hasattr(clf, 'standard_coef_') + assert not hasattr(clf, '_average_coef') + assert not hasattr(clf, '_average_intercept') + assert not hasattr(clf, '_standard_intercept') + assert not hasattr(clf, '_standard_coef') + + +# TODO: remove in 0.25 +@pytest.mark.parametrize('klass', [SGDClassifier, SGDRegressor]) +def test_sgd_deprecated_attr(klass): + est = klass(average=True, eta0=.01) + est.fit(X, Y) + + msg = "Attribute {} was deprecated" + for att in ['average_coef_', 'average_intercept_', + 'standard_coef_', 'standard_intercept_']: + with pytest.warns(FutureWarning, match=msg.format(att)): + getattr(est, att) @pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier, @@ -1568,11 +1577,6 @@ def test_multi_core_gridsearch_and_early_stopping(): assert search.best_score_ > 0.8 -@pytest.mark.skipif( - not hasattr(sp, "random"), - reason="this test uses scipy.random, that was introduced in version " - "0.17. This skip condition can be dropped as soon as we drop " - "support for scipy versions older than 0.17") @pytest.mark.parametrize("backend", ["loky", "multiprocessing", "threading"]) def test_SGDClassifier_fit_for_all_backends(backend): diff --git a/sklearn/manifold/_isomap.py b/sklearn/manifold/_isomap.py index 792c21ce51c2c..8a7fc3f85f425 100644 --- a/sklearn/manifold/_isomap.py +++ b/sklearn/manifold/_isomap.py @@ -140,13 +140,13 @@ def __init__(self, n_neighbors=5, n_components=2, eigen_solver='auto', self.metric_params = metric_params def _fit_transform(self, X): - self.nbrs_ = NearestNeighbors(n_neighbors=self.n_neighbors, algorithm=self.neighbors_algorithm, metric=self.metric, p=self.p, metric_params=self.metric_params, n_jobs=self.n_jobs) self.nbrs_.fit(X) + self.n_features_in_ = self.nbrs_.n_features_in_ self.kernel_pca_ = KernelPCA(n_components=self.n_components, kernel="precomputed", diff --git a/sklearn/manifold/_locally_linear.py b/sklearn/manifold/_locally_linear.py index b891c152e1a57..7b46d51df718d 100644 --- a/sklearn/manifold/_locally_linear.py +++ b/sklearn/manifold/_locally_linear.py @@ -143,12 +143,10 @@ def null_space(M, k, k_skip=1, eigen_solver='arpack', tol=1E-6, max_iter=100, Maximum number of iterations for 'arpack' method. Not used if eigen_solver=='dense' - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. Used when ``solver`` == 'arpack'. - + random_state : int, RandomState instance, default=None + Determines the random number generator when ``solver`` == 'arpack'. + Pass an int for reproducible results across multiple function calls. + See :term: `Glossary `. """ if eigen_solver == 'auto': if M.shape[0] > 200 and k + k_skip < 10: @@ -249,11 +247,10 @@ def locally_linear_embedding( Tolerance for modified LLE method. Only used if method == 'modified' - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. Used when ``solver`` == 'arpack'. + random_state : int, RandomState instance, default=None + Determines the random number generator when ``solver`` == 'arpack'. + Pass an int for reproducible results across multiple function calls. + See :term: `Glossary `. n_jobs : int or None, optional (default=None) The number of parallel jobs to run for neighbors search. @@ -581,11 +578,10 @@ class LocallyLinearEmbedding(TransformerMixin, algorithm to use for nearest neighbors search, passed to neighbors.NearestNeighbors instance - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. Used when ``eigen_solver`` == 'arpack'. + random_state : int, RandomState instance, default=None + Determines the random number generator when + ``eigen_solver`` == 'arpack'. Pass an int for reproducible results + across multiple function calls. See :term: `Glossary `. n_jobs : int or None, optional (default=None) The number of parallel jobs to run. @@ -656,7 +652,7 @@ def _fit_transform(self, X): n_jobs=self.n_jobs) random_state = check_random_state(self.random_state) - X = check_array(X, dtype=float) + X = self._validate_data(X, dtype=float) self.nbrs_.fit(X) self.embedding_, self.reconstruction_error_ = \ locally_linear_embedding( diff --git a/sklearn/manifold/_mds.py b/sklearn/manifold/_mds.py index 5238c67e93dfd..ca8c08ed69f98 100644 --- a/sklearn/manifold/_mds.py +++ b/sklearn/manifold/_mds.py @@ -48,11 +48,10 @@ def _smacof_single(dissimilarities, metric=True, n_components=2, init=None, Relative tolerance with respect to stress at which to declare convergence. - random_state : int, RandomState instance or None, optional, default: None - The generator used to initialize the centers. If int, random_state is - the seed used by the random number generator; If RandomState instance, - random_state is the random number generator; If None, the random number - generator is the RandomState instance used by `np.random`. + random_state : int, RandomState instance, default=None + Determines the random number generator used to initialize the centers. + Pass an int for reproducible results across multiple function calls. + See :term: `Glossary `. Returns ------- @@ -195,11 +194,10 @@ def smacof(dissimilarities, metric=True, n_components=2, init=None, n_init=8, Relative tolerance with respect to stress at which to declare convergence. - random_state : int, RandomState instance or None, optional, default: None - The generator used to initialize the centers. If int, random_state is - the seed used by the random number generator; If RandomState instance, - random_state is the random number generator; If None, the random number - generator is the RandomState instance used by `np.random`. + random_state : int, RandomState instance, default=None + Determines the random number generator used to initialize the centers. + Pass an int for reproducible results across multiple function calls. + See :term: `Glossary `. return_n_iter : bool, optional, default: False Whether or not to return the number of iterations. @@ -311,11 +309,10 @@ class MDS(BaseEstimator): ``-1`` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance or None, optional, default: None - The generator used to initialize the centers. If int, random_state is - the seed used by the random number generator; If RandomState instance, - random_state is the random number generator; If None, the random number - generator is the RandomState instance used by `np.random`. + random_state : int, RandomState instance, default=None + Determines the random number generator used to initialize the centers. + Pass an int for reproducible results across multiple function calls. + See :term: `Glossary `. dissimilarity : 'euclidean' | 'precomputed', optional, default: 'euclidean' Dissimilarity measure to use: @@ -414,7 +411,7 @@ def fit_transform(self, X, y=None, init=None): algorithm. By default, the algorithm is initialized with a randomly chosen array. """ - X = check_array(X) + X = self._validate_data(X) if X.shape[0] == X.shape[1] and self.dissimilarity != "precomputed": warnings.warn("The MDS API has changed. ``fit`` now constructs an" " dissimilarity matrix from data. To use a custom " diff --git a/sklearn/manifold/_spectral_embedding.py b/sklearn/manifold/_spectral_embedding.py index 9d52a9787425c..caac2236e1dd6 100644 --- a/sklearn/manifold/_spectral_embedding.py +++ b/sklearn/manifold/_spectral_embedding.py @@ -168,13 +168,11 @@ def spectral_embedding(adjacency, n_components=8, eigen_solver=None, to be installed. It can be faster on very large, sparse problems, but may also lead to instabilities. - random_state : int, RandomState instance or None, optional, default: None - A pseudo random number generator used for the initialization of the - lobpcg eigenvectors decomposition. If int, random_state is the seed - used by the random number generator; If RandomState instance, - random_state is the random number generator; If None, the random number - generator is the RandomState instance used by `np.random`. Used when - ``solver`` == 'amg'. + random_state : int, RandomState instance, default=None + Determines the random number generator used for the initialization of + the lobpcg eigenvectors decomposition when ``solver`` == 'amg'. Pass + an int for reproducible results across multiple function calls. + See :term: `Glossary `. eigen_tol : float, optional, default=0.0 Stopping criterion for eigendecomposition of the Laplacian matrix @@ -384,13 +382,11 @@ class SpectralEmbedding(BaseEstimator): gamma : float, optional, default : 1/n_features Kernel coefficient for rbf kernel. - random_state : int, RandomState instance or None, optional, default: None - A pseudo random number generator used for the initialization of the - lobpcg eigenvectors. If int, random_state is the seed used by the - random number generator; If RandomState instance, random_state is the - random number generator; If None, the random number generator is the - RandomState instance used by `np.random`. Used when ``solver`` == - 'amg'. + random_state : int, RandomState instance, default=None + Determines the random number generator used for the initialization of + the lobpcg eigenvectors when ``solver`` == 'amg'. Pass an int for + reproducible results across multiple function calls. + See :term: `Glossary `. eigen_solver : {None, 'arpack', 'lobpcg', or 'amg'} The eigenvalue decomposition strategy to use. AMG requires pyamg @@ -535,8 +531,8 @@ def fit(self, X, y=None): Returns the instance itself. """ - X = check_array(X, accept_sparse='csr', ensure_min_samples=2, - estimator=self) + X = self._validate_data(X, accept_sparse='csr', ensure_min_samples=2, + estimator=self) random_state = check_random_state(self.random_state) if isinstance(self.affinity, str): diff --git a/sklearn/manifold/_t_sne.py b/sklearn/manifold/_t_sne.py index f2d15f54e8f40..d0c9e4e509a73 100644 --- a/sklearn/manifold/_t_sne.py +++ b/sklearn/manifold/_t_sne.py @@ -555,12 +555,11 @@ class TSNE(BaseEstimator): verbose : int, optional (default: 0) Verbosity level. - random_state : int, RandomState instance or None, optional (default: None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. Note that different initializations might result in - different local minima of the cost function. + random_state : int, RandomState instance, default=None + Determines the random number generator. Pass an int for reproducible + results across multiple function calls. Note that different + initializations might result in different local minima of the cost + function. See :term: `Glossary `. method : string (default: 'barnes_hut') By default the gradient calculation algorithm uses Barnes-Hut @@ -662,11 +661,12 @@ def _fit(self, X, skip_num_points=0): if self.angle < 0.0 or self.angle > 1.0: raise ValueError("'angle' must be between 0.0 - 1.0") if self.method == 'barnes_hut': - X = check_array(X, accept_sparse=['csr'], ensure_min_samples=2, - dtype=[np.float32, np.float64]) + X = self._validate_data(X, accept_sparse=['csr'], + ensure_min_samples=2, + dtype=[np.float32, np.float64]) else: - X = check_array(X, accept_sparse=['csr', 'csc', 'coo'], - dtype=[np.float32, np.float64]) + X = self._validate_data(X, accept_sparse=['csr', 'csc', 'coo'], + dtype=[np.float32, np.float64]) if self.metric == "precomputed": if isinstance(self.init, str) and self.init == 'pca': raise ValueError("The parameter init=\"pca\" cannot be " diff --git a/sklearn/manifold/tests/test_spectral_embedding.py b/sklearn/manifold/tests/test_spectral_embedding.py index 295367a422f04..3d9d87af3a09a 100644 --- a/sklearn/manifold/tests/test_spectral_embedding.py +++ b/sklearn/manifold/tests/test_spectral_embedding.py @@ -32,18 +32,13 @@ cluster_std=1., random_state=42) -def _check_with_col_sign_flipping(A, B, tol=0.0): +def _assert_equal_with_sign_flipping(A, B, tol=0.0): """ Check array A and B are equal with possible sign flipping on each columns""" - sign = True - for column_idx in range(A.shape[1]): - sign = sign and ((((A[:, column_idx] - - B[:, column_idx]) ** 2).mean() <= tol ** 2) or - (((A[:, column_idx] + - B[:, column_idx]) ** 2).mean() <= tol ** 2)) - if not sign: - return False - return True + tol_squared = tol ** 2 + for A_col, B_col in zip(A.T, B.T): + assert (np.max((A_col - B_col) ** 2) <= tol_squared or + np.max((A_col + B_col) ** 2) <= tol_squared) def test_sparse_graph_connected_component(): @@ -139,7 +134,7 @@ def test_spectral_embedding_precomputed_affinity(X, seed=36): embed_rbf = se_rbf.fit_transform(X) assert_array_almost_equal( se_precomp.affinity_matrix_, se_rbf.affinity_matrix_) - assert _check_with_col_sign_flipping(embed_precomp, embed_rbf, 0.05) + _assert_equal_with_sign_flipping(embed_precomp, embed_rbf, 0.05) def test_precomputed_nearest_neighbors_filtering(): @@ -178,9 +173,13 @@ def test_spectral_embedding_callable_affinity(X, seed=36): assert_array_almost_equal( se_callable.affinity_matrix_, se_rbf.affinity_matrix_) assert_array_almost_equal(kern, se_rbf.affinity_matrix_) - assert _check_with_col_sign_flipping(embed_rbf, embed_callable, 0.05) + _assert_equal_with_sign_flipping(embed_rbf, embed_callable, 0.05) +# TODO: Remove when pyamg does replaces sp.rand call with np.random.rand +# https://github.com/scikit-learn/scikit-learn/issues/15913 +@pytest.mark.filterwarnings( + "ignore:scipy.rand is deprecated:DeprecationWarning:pyamg.*") def test_spectral_embedding_amg_solver(seed=36): # Test spectral embedding with amg solver pytest.importorskip('pyamg') @@ -193,7 +192,7 @@ def test_spectral_embedding_amg_solver(seed=36): random_state=np.random.RandomState(seed)) embed_amg = se_amg.fit_transform(S) embed_arpack = se_arpack.fit_transform(S) - assert _check_with_col_sign_flipping(embed_amg, embed_arpack, 1e-5) + _assert_equal_with_sign_flipping(embed_amg, embed_arpack, 1e-5) # same with special case in which amg is not actually used # regression test for #10715 @@ -208,31 +207,34 @@ def test_spectral_embedding_amg_solver(seed=36): se_arpack.affinity = "precomputed" embed_amg = se_amg.fit_transform(affinity) embed_arpack = se_arpack.fit_transform(affinity) - assert _check_with_col_sign_flipping(embed_amg, embed_arpack, 1e-5) + _assert_equal_with_sign_flipping(embed_amg, embed_arpack, 1e-5) -def test_spectral_embedding_amg_solver_failure(seed=36): - # Test spectral embedding with amg solver failure, see issue #13393 +# TODO: Remove filterwarnings when pyamg does replaces sp.rand call with +# np.random.rand: +# https://github.com/scikit-learn/scikit-learn/issues/15913 +@pytest.mark.filterwarnings( + "ignore:scipy.rand is deprecated:DeprecationWarning:pyamg.*") +def test_spectral_embedding_amg_solver_failure(): + # Non-regression test for amg solver failure (issue #13393 on github) pytest.importorskip('pyamg') - - # The generated graph below is NOT fully connected if n_neighbors=3 - n_samples = 200 - n_clusters = 3 - n_features = 3 - centers = np.eye(n_clusters, n_features) - S, true_labels = make_blobs(n_samples=n_samples, centers=centers, - cluster_std=1., random_state=42) - - se_amg0 = SpectralEmbedding(n_components=3, affinity="nearest_neighbors", - eigen_solver="amg", n_neighbors=3, - random_state=np.random.RandomState(seed)) - embed_amg0 = se_amg0.fit_transform(S) - - for i in range(10): - se_amg0.set_params(random_state=np.random.RandomState(seed + 1)) - embed_amg1 = se_amg0.fit_transform(S) - - assert _check_with_col_sign_flipping(embed_amg0, embed_amg1, 0.05) + seed = 36 + num_nodes = 100 + X = sparse.rand(num_nodes, num_nodes, density=0.1, random_state=seed) + upper = sparse.triu(X) - sparse.diags(X.diagonal()) + sym_matrix = upper + upper.T + embedding = spectral_embedding(sym_matrix, + n_components=10, + eigen_solver='amg', + random_state=0) + + # Check that the learned embedding is stable w.r.t. random solver init: + for i in range(3): + new_embedding = spectral_embedding(sym_matrix, + n_components=10, + eigen_solver='amg', + random_state=i + 1) + _assert_equal_with_sign_flipping(embedding, new_embedding, tol=0.05) @pytest.mark.filterwarnings("ignore:the behavior of nmi will " diff --git a/sklearn/manifold/tests/test_t_sne.py b/sklearn/manifold/tests/test_t_sne.py index 15ce1fa6f2482..9486bbd4a96f5 100644 --- a/sklearn/manifold/tests/test_t_sne.py +++ b/sklearn/manifold/tests/test_t_sne.py @@ -772,11 +772,11 @@ def test_uniform_grid(method): we re-run t-SNE from the final point when the convergence is not good enough. """ - seeds = [0, 1, 2] + seeds = range(3) n_iter = 500 for seed in seeds: tsne = TSNE(n_components=2, init='random', random_state=seed, - perplexity=20, n_iter=n_iter, method=method) + perplexity=50, n_iter=n_iter, method=method) Y = tsne.fit_transform(X_2d_grid) try_name = "{}_{}".format(method, seed) diff --git a/sklearn/metrics/__init__.py b/sklearn/metrics/__init__.py index ac6162b924a90..8bcb047ec8161 100644 --- a/sklearn/metrics/__init__.py +++ b/sklearn/metrics/__init__.py @@ -24,7 +24,6 @@ from ._classification import fbeta_score from ._classification import hamming_loss from ._classification import hinge_loss -from ._classification import jaccard_similarity_score from ._classification import jaccard_score from ._classification import log_loss from ._classification import matthews_corrcoef @@ -48,7 +47,6 @@ from .cluster import silhouette_samples from .cluster import silhouette_score from .cluster import calinski_harabasz_score -from .cluster import calinski_harabaz_score from .cluster import v_measure_score from .cluster import davies_bouldin_score @@ -82,6 +80,9 @@ from ._plot.precision_recall_curve import plot_precision_recall_curve from ._plot.precision_recall_curve import PrecisionRecallDisplay +from ._plot.confusion_matrix import plot_confusion_matrix +from ._plot.confusion_matrix import ConfusionMatrixDisplay + __all__ = [ 'accuracy_score', @@ -90,13 +91,13 @@ 'auc', 'average_precision_score', 'balanced_accuracy_score', - 'calinski_harabaz_score', 'calinski_harabasz_score', 'check_scoring', 'classification_report', 'cluster', 'cohen_kappa_score', 'completeness_score', + 'ConfusionMatrixDisplay', 'confusion_matrix', 'consensus_score', 'coverage_error', @@ -113,7 +114,6 @@ 'homogeneity_completeness_v_measure', 'homogeneity_score', 'jaccard_score', - 'jaccard_similarity_score', 'label_ranking_average_precision_score', 'label_ranking_loss', 'log_loss', @@ -137,6 +137,7 @@ 'pairwise_distances_argmin_min', 'pairwise_distances_chunked', 'pairwise_kernels', + 'plot_confusion_matrix', 'plot_precision_recall_curve', 'plot_roc_curve', 'PrecisionRecallDisplay', diff --git a/sklearn/metrics/_classification.py b/sklearn/metrics/_classification.py index 04d0a009df4b0..a916bbe1dd955 100644 --- a/sklearn/metrics/_classification.py +++ b/sklearn/metrics/_classification.py @@ -193,8 +193,9 @@ def accuracy_score(y_true, y_pred, normalize=True, sample_weight=None): return _weighted_sum(score, sample_weight, normalize) -def confusion_matrix(y_true, y_pred, labels=None, sample_weight=None): - """Compute confusion matrix to evaluate the accuracy of a classification +def confusion_matrix(y_true, y_pred, labels=None, sample_weight=None, + normalize=None): + """Compute confusion matrix to evaluate the accuracy of a classification. By definition a confusion matrix :math:`C` is such that :math:`C_{i, j}` is equal to the number of observations known to be in group :math:`i` and @@ -208,25 +209,33 @@ def confusion_matrix(y_true, y_pred, labels=None, sample_weight=None): Parameters ---------- - y_true : array, shape = [n_samples] + y_true : array-like of shape (n_samples,) Ground truth (correct) target values. - y_pred : array, shape = [n_samples] + y_pred : array-like of shape (n_samples,) Estimated targets as returned by a classifier. - labels : array, shape = [n_classes], optional + labels : array-like of shape (n_classes), default=None List of labels to index the matrix. This may be used to reorder or select a subset of labels. - If none is given, those that appear at least once + If ``None`` is given, those that appear at least once in ``y_true`` or ``y_pred`` are used in sorted order. sample_weight : array-like of shape (n_samples,), default=None Sample weights. + normalize : {'true', 'pred', 'all'}, default=None + Normalizes confusion matrix over the true (rows), predicted (columns) + conditions or all the population. If None, confusion matrix will not be + normalized. + Returns ------- C : ndarray of shape (n_classes, n_classes) - Confusion matrix + Confusion matrix whose i-th row and j-th + column entry indicates the number of + samples with true label being i-th class + and prediced label being j-th class. References ---------- @@ -267,7 +276,12 @@ def confusion_matrix(y_true, y_pred, labels=None, sample_weight=None): labels = unique_labels(y_true, y_pred) else: labels = np.asarray(labels) - if np.all([l not in y_true for l in labels]): + n_labels = labels.size + if n_labels == 0: + raise ValueError("'labels' should contains at least one label.") + elif y_true.size == 0: + return np.zeros((n_labels, n_labels), dtype=np.int) + elif np.all([l not in y_true for l in labels]): raise ValueError("At least one label specified must be in y_true") if sample_weight is None: @@ -277,6 +291,10 @@ def confusion_matrix(y_true, y_pred, labels=None, sample_weight=None): check_consistent_length(y_true, y_pred, sample_weight) + if normalize not in ['true', 'pred', 'all', None]: + raise ValueError("normalize must be one of {'true', 'pred', " + "'all', None}") + n_labels = labels.size label_to_ind = {y: x for x, y in enumerate(labels)} # convert yt, yp into index @@ -296,11 +314,20 @@ def confusion_matrix(y_true, y_pred, labels=None, sample_weight=None): else: dtype = np.float64 - CM = coo_matrix((sample_weight, (y_true, y_pred)), + cm = coo_matrix((sample_weight, (y_true, y_pred)), shape=(n_labels, n_labels), dtype=dtype, ).toarray() - return CM + with np.errstate(all='ignore'): + if normalize == 'true': + cm = cm / cm.sum(axis=1, keepdims=True) + elif normalize == 'pred': + cm = cm / cm.sum(axis=0, keepdims=True) + elif normalize == 'all': + cm = cm / cm.sum() + cm = np.nan_to_num(cm) + + return cm def multilabel_confusion_matrix(y_true, y_pred, sample_weight=None, @@ -586,80 +613,6 @@ class labels [2]_. return 1 - k -def jaccard_similarity_score(y_true, y_pred, normalize=True, - sample_weight=None): - """Jaccard similarity coefficient score - - .. deprecated:: 0.21 - This is deprecated to be removed in 0.23, since its handling of - binary and multiclass inputs was broken. `jaccard_score` has an API - that is consistent with precision_score, f_score, etc. - - Read more in the :ref:`User Guide `. - - Parameters - ---------- - y_true : 1d array-like, or label indicator array / sparse matrix - Ground truth (correct) labels. - - y_pred : 1d array-like, or label indicator array / sparse matrix - Predicted labels, as returned by a classifier. - - normalize : bool, optional (default=True) - If ``False``, return the sum of the Jaccard similarity coefficient - over the sample set. Otherwise, return the average of Jaccard - similarity coefficient. - - sample_weight : array-like of shape (n_samples,), default=None - Sample weights. - - Returns - ------- - score : float - If ``normalize == True``, return the average Jaccard similarity - coefficient, else it returns the sum of the Jaccard similarity - coefficient over the sample set. - - The best performance is 1 with ``normalize == True`` and the number - of samples with ``normalize == False``. - - See also - -------- - accuracy_score, hamming_loss, zero_one_loss - - Notes - ----- - In binary and multiclass classification, this function is equivalent - to the ``accuracy_score``. It differs in the multilabel classification - problem. - - References - ---------- - .. [1] `Wikipedia entry for the Jaccard index - `_ - """ - warnings.warn('jaccard_similarity_score has been deprecated and replaced ' - 'with jaccard_score. It will be removed in version 0.23. ' - 'This implementation has surprising behavior for binary ' - 'and multiclass classification tasks.', - FutureWarning) - - # Compute accuracy for each possible representation - y_type, y_true, y_pred = _check_targets(y_true, y_pred) - check_consistent_length(y_true, y_pred, sample_weight) - if y_type.startswith('multilabel'): - with np.errstate(divide='ignore', invalid='ignore'): - # oddly, we may get an "invalid" rather than a "divide" error here - pred_or_true = count_nonzero(y_true + y_pred, axis=1) - pred_and_true = count_nonzero(y_true.multiply(y_pred), axis=1) - score = pred_and_true / pred_or_true - score[pred_or_true == 0.0] = 1.0 - else: - score = y_true == y_pred - - return _weighted_sum(score, sample_weight, normalize) - - def jaccard_score(y_true, y_pred, labels=None, pos_label=1, average='binary', sample_weight=None): """Jaccard similarity coefficient score @@ -1411,7 +1364,7 @@ def precision_recall_fscore_support(y_true, y_pred, beta=1.0, labels=None, fbeta_score : float (if average is not None) or array of float, shape =\ [n_unique_labels] - support : int (if average is not None) or array of int, shape =\ + support : None (if average is not None) or array of int, shape =\ [n_unique_labels] The number of occurrences of each label in ``y_true``. @@ -1851,7 +1804,7 @@ def balanced_accuracy_score(y_true, y_pred, sample_weight=None, def classification_report(y_true, y_pred, labels=None, target_names=None, sample_weight=None, digits=2, output_dict=False, zero_division="warn"): - """Build a text report showing the main classification metrics + """Build a text report showing the main classification metrics. Read more in the :ref:`User Guide `. @@ -1901,10 +1854,10 @@ def classification_report(y_true, y_pred, labels=None, target_names=None, The reported averages include macro average (averaging the unweighted mean per label), weighted average (averaging the support-weighted mean - per label), sample average (only for multilabel classification) and - micro average (averaging the total true positives, false negatives and - false positives) it is only shown for multi-label or multi-class - with a subset of classes because it is accuracy otherwise. + per label), and sample average (only for multilabel classification). + Micro average (averaging the total true positives, false negatives and + false positives) is only shown for multi-label or multi-class + with a subset of classes, because it corresponds to accuracy otherwise. See also :func:`precision_recall_fscore_support` for more details on averages. @@ -2019,7 +1972,8 @@ class 2 1.00 0.67 0.80 3 # compute averages with specified averaging method avg_p, avg_r, avg_f1, _ = precision_recall_fscore_support( y_true, y_pred, labels=labels, - average=average, sample_weight=sample_weight) + average=average, sample_weight=sample_weight, + zero_division=zero_division) avg = [avg_p, avg_r, avg_f1, np.sum(s)] if output_dict: @@ -2045,7 +1999,7 @@ class 2 1.00 0.67 0.80 3 return report -def hamming_loss(y_true, y_pred, labels=None, sample_weight=None): +def hamming_loss(y_true, y_pred, sample_weight=None): """Compute the average Hamming loss. The Hamming loss is the fraction of labels that are incorrectly predicted. @@ -2060,17 +2014,6 @@ def hamming_loss(y_true, y_pred, labels=None, sample_weight=None): y_pred : 1d array-like, or label indicator array / sparse matrix Predicted labels, as returned by a classifier. - labels : array, shape = [n_labels], optional (default='deprecated') - Integer array of labels. If not provided, labels will be inferred - from y_true and y_pred. - - .. versionadded:: 0.18 - .. deprecated:: 0.21 - This parameter ``labels`` is deprecated in version 0.21 and will - be removed in version 0.23. Hamming loss uses ``y_true.shape[1]`` - for the number of labels when y_true is binary label indicators, - so it is unnecessary for the user to specify. - sample_weight : array-like of shape (n_samples,), default=None Sample weights. @@ -2130,12 +2073,6 @@ def hamming_loss(y_true, y_pred, labels=None, sample_weight=None): y_type, y_true, y_pred = _check_targets(y_true, y_pred) check_consistent_length(y_true, y_pred, sample_weight) - if labels is not None: - warnings.warn("The labels parameter is unused. It was" - " deprecated in version 0.21 and" - " will be removed in version 0.23", - FutureWarning) - if sample_weight is None: weight_average = 1. else: @@ -2159,8 +2096,9 @@ def log_loss(y_true, y_pred, eps=1e-15, normalize=True, sample_weight=None, This is the loss function used in (multinomial) logistic regression and extensions of it such as neural networks, defined as the negative - log-likelihood of the true labels given a probabilistic classifier's - predictions. The log loss is only defined for two or more labels. + log-likelihood of a logistic model that returns ``y_pred`` probabilities + for its training data ``y_true``. + The log loss is only defined for two or more labels. For a single sample with true label yt in {0,1} and estimated probability yp that yt = 1, the log loss is @@ -2400,6 +2338,7 @@ def hinge_loss(y_true, pred_decision, labels=None, sample_weight=None): def brier_score_loss(y_true, y_prob, sample_weight=None, pos_label=None): """Compute the Brier score. + The smaller the Brier score, the better, hence the naming with "loss". Across all items in a set N predictions, the Brier score measures the mean squared difference between (1) the predicted probability assigned diff --git a/sklearn/metrics/_plot/confusion_matrix.py b/sklearn/metrics/_plot/confusion_matrix.py new file mode 100644 index 0000000000000..96d99adfe7386 --- /dev/null +++ b/sklearn/metrics/_plot/confusion_matrix.py @@ -0,0 +1,223 @@ +from itertools import product + +import numpy as np + +from .. import confusion_matrix +from ...utils import check_matplotlib_support +from ...base import is_classifier + + +class ConfusionMatrixDisplay: + """Confusion Matrix visualization. + + It is recommend to use :func:`~sklearn.metrics.plot_confusion_matrix` to + create a :class:`ConfusionMatrixDisplay`. All parameters are stored as + attributes. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + confusion_matrix : ndarray of shape (n_classes, n_classes) + Confusion matrix. + + display_labels : ndarray of shape (n_classes,) + Display labels for plot. + + Attributes + ---------- + im_ : matplotlib AxesImage + Image representing the confusion matrix. + + text_ : ndarray of shape (n_classes, n_classes), dtype=matplotlib Text, \ + or None + Array of matplotlib axes. `None` if `include_values` is false. + + ax_ : matplotlib Axes + Axes with confusion matrix. + + figure_ : matplotlib Figure + Figure containing the confusion matrix. + """ + def __init__(self, confusion_matrix, display_labels): + self.confusion_matrix = confusion_matrix + self.display_labels = display_labels + + def plot(self, include_values=True, cmap='viridis', + xticks_rotation='horizontal', values_format=None, ax=None): + """Plot visualization. + + Parameters + ---------- + include_values : bool, default=True + Includes values in confusion matrix. + + cmap : str or matplotlib Colormap, default='viridis' + Colormap recognized by matplotlib. + + xticks_rotation : {'vertical', 'horizontal'} or float, \ + default='horizontal' + Rotation of xtick labels. + + values_format : str, default=None + Format specification for values in confusion matrix. If `None`, + the format specification is 'd' or '.2g' whichever is shorter. + + ax : matplotlib axes, default=None + Axes object to plot on. If `None`, a new figure and axes is + created. + + Returns + ------- + display : :class:`~sklearn.metrics.ConfusionMatrixDisplay` + """ + check_matplotlib_support("ConfusionMatrixDisplay.plot") + import matplotlib.pyplot as plt + + if ax is None: + fig, ax = plt.subplots() + else: + fig = ax.figure + + cm = self.confusion_matrix + n_classes = cm.shape[0] + self.im_ = ax.imshow(cm, interpolation='nearest', cmap=cmap) + self.text_ = None + cmap_min, cmap_max = self.im_.cmap(0), self.im_.cmap(256) + + if include_values: + self.text_ = np.empty_like(cm, dtype=object) + + # print text with appropriate color depending on background + thresh = (cm.max() + cm.min()) / 2.0 + + for i, j in product(range(n_classes), range(n_classes)): + color = cmap_max if cm[i, j] < thresh else cmap_min + + if values_format is None: + text_cm = format(cm[i, j], '.2g') + if cm.dtype.kind != 'f': + text_d = format(cm[i, j], 'd') + if len(text_d) < len(text_cm): + text_cm = text_d + else: + text_cm = format(cm[i, j], values_format) + + self.text_[i, j] = ax.text( + j, i, text_cm, + ha="center", va="center", + color=color) + + fig.colorbar(self.im_, ax=ax) + ax.set(xticks=np.arange(n_classes), + yticks=np.arange(n_classes), + xticklabels=self.display_labels, + yticklabels=self.display_labels, + ylabel="True label", + xlabel="Predicted label") + + ax.set_ylim((n_classes - 0.5, -0.5)) + plt.setp(ax.get_xticklabels(), rotation=xticks_rotation) + + self.figure_ = fig + self.ax_ = ax + return self + + +def plot_confusion_matrix(estimator, X, y_true, labels=None, + sample_weight=None, normalize=None, + display_labels=None, include_values=True, + xticks_rotation='horizontal', + values_format=None, + cmap='viridis', ax=None): + """Plot Confusion Matrix. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + estimator : estimator instance + Trained classifier. + + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Input values. + + y : array-like of shape (n_samples,) + Target values. + + labels : array-like of shape (n_classes,), default=None + List of labels to index the matrix. This may be used to reorder or + select a subset of labels. If `None` is given, those that appear at + least once in `y_true` or `y_pred` are used in sorted order. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + + normalize : {'true', 'pred', 'all'}, default=None + Normalizes confusion matrix over the true (rows), predicted (columns) + conditions or all the population. If None, confusion matrix will not be + normalized. + + display_labels : array-like of shape (n_classes,), default=None + Target names used for plotting. By default, `labels` will be used if + it is defined, otherwise the unique labels of `y_true` and `y_pred` + will be used. + + include_values : bool, default=True + Includes values in confusion matrix. + + xticks_rotation : {'vertical', 'horizontal'} or float, \ + default='horizontal' + Rotation of xtick labels. + + values_format : str, default=None + Format specification for values in confusion matrix. If `None`, + the format specification is 'd' or '.2g' whichever is shorter. + + cmap : str or matplotlib Colormap, default='viridis' + Colormap recognized by matplotlib. + + ax : matplotlib Axes, default=None + Axes object to plot on. If `None`, a new figure and axes is + created. + + Returns + ------- + display : :class:`~sklearn.metrics.ConfusionMatrixDisplay` + + Examples + -------- + >>> import matplotlib.pyplot as plt # doctest: +SKIP + >>> from sklearn.datasets import make_classification + >>> from sklearn.metrics import plot_confusion_matrix + >>> from sklearn.model_selection import train_test_split + >>> from sklearn.svm import SVC + >>> X, y = make_classification(random_state=0) + >>> X_train, X_test, y_train, y_test = train_test_split( + ... X, y, random_state=0) + >>> clf = SVC(random_state=0) + >>> clf.fit(X_train, y_train) + SVC(random_state=0) + >>> plot_confusion_matrix(clf, X_test, y_test) # doctest: +SKIP + >>> plt.show() # doctest: +SKIP + """ + check_matplotlib_support("plot_confusion_matrix") + + if not is_classifier(estimator): + raise ValueError("plot_confusion_matrix only supports classifiers") + + y_pred = estimator.predict(X) + cm = confusion_matrix(y_true, y_pred, sample_weight=sample_weight, + labels=labels, normalize=normalize) + + if display_labels is None: + if labels is None: + display_labels = estimator.classes_ + else: + display_labels = labels + + disp = ConfusionMatrixDisplay(confusion_matrix=cm, + display_labels=display_labels) + return disp.plot(include_values=include_values, + cmap=cmap, ax=ax, xticks_rotation=xticks_rotation, + values_format=values_format) diff --git a/sklearn/metrics/_plot/precision_recall_curve.py b/sklearn/metrics/_plot/precision_recall_curve.py index d2b84059c3c0e..a83fbe5acc60a 100644 --- a/sklearn/metrics/_plot/precision_recall_curve.py +++ b/sklearn/metrics/_plot/precision_recall_curve.py @@ -97,7 +97,7 @@ def plot(self, ax=None, name=None, **kwargs): def plot_precision_recall_curve(estimator, X, y, sample_weight=None, response_method="auto", name=None, ax=None, **kwargs): - """Plot Precision Recall Curve for binary classifers. + """Plot Precision Recall Curve for binary classifiers. Extra keyword arguments will be passed to matplotlib's `plot`. @@ -141,7 +141,7 @@ def plot_precision_recall_curve(estimator, X, y, """ check_matplotlib_support("plot_precision_recall_curve") - classification_error = ("{} should be a binary classifer".format( + classification_error = ("{} should be a binary classifier".format( estimator.__class__.__name__)) if not is_classifier(estimator): raise ValueError(classification_error) @@ -161,7 +161,11 @@ def plot_precision_recall_curve(estimator, X, y, pos_label=pos_label, sample_weight=sample_weight) average_precision = average_precision_score(y, y_pred, + pos_label=pos_label, sample_weight=sample_weight) - viz = PrecisionRecallDisplay(precision, recall, average_precision, - estimator.__class__.__name__) + name = name if name is not None else estimator.__class__.__name__ + viz = PrecisionRecallDisplay( + precision=precision, recall=recall, + average_precision=average_precision, estimator_name=name + ) return viz.plot(ax=ax, name=name, **kwargs) diff --git a/sklearn/metrics/_plot/roc_curve.py b/sklearn/metrics/_plot/roc_curve.py index c86a7d5ceaf3a..fb76691ff37d1 100644 --- a/sklearn/metrics/_plot/roc_curve.py +++ b/sklearn/metrics/_plot/roc_curve.py @@ -165,8 +165,9 @@ def plot_roc_curve(estimator, X, y, sample_weight=None, """ check_matplotlib_support('plot_roc_curve') - classification_error = ("{} should be a binary classifer".format( - estimator.__class__.__name__)) + classification_error = ( + "{} should be a binary classifier".format(estimator.__class__.__name__) + ) if not is_classifier(estimator): raise ValueError(classification_error) @@ -185,5 +186,8 @@ def plot_roc_curve(estimator, X, y, sample_weight=None, sample_weight=sample_weight, drop_intermediate=drop_intermediate) roc_auc = auc(fpr, tpr) - viz = RocCurveDisplay(fpr, tpr, roc_auc, estimator.__class__.__name__) + name = estimator.__class__.__name__ if name is None else name + viz = RocCurveDisplay( + fpr=fpr, tpr=tpr, roc_auc=roc_auc, estimator_name=name + ) return viz.plot(ax=ax, name=name, **kwargs) diff --git a/sklearn/metrics/_plot/tests/test_plot_confusion_matrix.py b/sklearn/metrics/_plot/tests/test_plot_confusion_matrix.py new file mode 100644 index 0000000000000..b8a24ae15f1e5 --- /dev/null +++ b/sklearn/metrics/_plot/tests/test_plot_confusion_matrix.py @@ -0,0 +1,282 @@ +import pytest +import numpy as np +from numpy.testing import assert_allclose +from numpy.testing import assert_array_equal + +from sklearn.compose import make_column_transformer +from sklearn.datasets import make_classification +from sklearn.exceptions import NotFittedError +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler +from sklearn.svm import SVC, SVR + +from sklearn.metrics import confusion_matrix +from sklearn.metrics import plot_confusion_matrix +from sklearn.metrics import ConfusionMatrixDisplay + + +# TODO: Remove when https://github.com/numpy/numpy/issues/14397 is resolved +pytestmark = pytest.mark.filterwarnings( + "ignore:In future, it will be an error for 'np.bool_':DeprecationWarning:" + "matplotlib.*") + + +@pytest.fixture(scope="module") +def n_classes(): + return 5 + + +@pytest.fixture(scope="module") +def data(n_classes): + X, y = make_classification(n_samples=100, n_informative=5, + n_classes=n_classes, random_state=0) + return X, y + + +@pytest.fixture(scope="module") +def fitted_clf(data): + return SVC(kernel='linear', C=0.01).fit(*data) + + +@pytest.fixture(scope="module") +def y_pred(data, fitted_clf): + X, _ = data + return fitted_clf.predict(X) + + +def test_error_on_regressor(pyplot, data): + X, y = data + est = SVR().fit(X, y) + + msg = "plot_confusion_matrix only supports classifiers" + with pytest.raises(ValueError, match=msg): + plot_confusion_matrix(est, X, y) + + +def test_error_on_invalid_option(pyplot, fitted_clf, data): + X, y = data + msg = (r"normalize must be one of \{'true', 'pred', 'all', " + r"None\}") + + with pytest.raises(ValueError, match=msg): + plot_confusion_matrix(fitted_clf, X, y, normalize='invalid') + + +@pytest.mark.parametrize("with_labels", [True, False]) +@pytest.mark.parametrize("with_display_labels", [True, False]) +def test_plot_confusion_matrix_custom_labels(pyplot, data, y_pred, fitted_clf, + n_classes, with_labels, + with_display_labels): + X, y = data + ax = pyplot.gca() + labels = [2, 1, 0, 3, 4] if with_labels else None + display_labels = ['b', 'd', 'a', 'e', 'f'] if with_display_labels else None + + cm = confusion_matrix(y, y_pred, labels=labels) + disp = plot_confusion_matrix(fitted_clf, X, y, + ax=ax, display_labels=display_labels, + labels=labels) + + assert_allclose(disp.confusion_matrix, cm) + + if with_display_labels: + expected_display_labels = display_labels + elif with_labels: + expected_display_labels = labels + else: + expected_display_labels = list(range(n_classes)) + + expected_display_labels_str = [str(name) + for name in expected_display_labels] + + x_ticks = [tick.get_text() for tick in disp.ax_.get_xticklabels()] + y_ticks = [tick.get_text() for tick in disp.ax_.get_yticklabels()] + + assert_array_equal(disp.display_labels, expected_display_labels) + assert_array_equal(x_ticks, expected_display_labels_str) + assert_array_equal(y_ticks, expected_display_labels_str) + + +@pytest.mark.parametrize("normalize", ['true', 'pred', 'all', None]) +@pytest.mark.parametrize("include_values", [True, False]) +def test_plot_confusion_matrix(pyplot, data, y_pred, n_classes, fitted_clf, + normalize, include_values): + X, y = data + ax = pyplot.gca() + cmap = 'plasma' + cm = confusion_matrix(y, y_pred) + disp = plot_confusion_matrix(fitted_clf, X, y, + normalize=normalize, + cmap=cmap, ax=ax, + include_values=include_values) + + assert disp.ax_ == ax + + if normalize == 'true': + cm = cm / cm.sum(axis=1, keepdims=True) + elif normalize == 'pred': + cm = cm / cm.sum(axis=0, keepdims=True) + elif normalize == 'all': + cm = cm / cm.sum() + + assert_allclose(disp.confusion_matrix, cm) + import matplotlib as mpl + assert isinstance(disp.im_, mpl.image.AxesImage) + assert disp.im_.get_cmap().name == cmap + assert isinstance(disp.ax_, pyplot.Axes) + assert isinstance(disp.figure_, pyplot.Figure) + + assert disp.ax_.get_ylabel() == "True label" + assert disp.ax_.get_xlabel() == "Predicted label" + + x_ticks = [tick.get_text() for tick in disp.ax_.get_xticklabels()] + y_ticks = [tick.get_text() for tick in disp.ax_.get_yticklabels()] + + expected_display_labels = list(range(n_classes)) + + expected_display_labels_str = [str(name) + for name in expected_display_labels] + + assert_array_equal(disp.display_labels, expected_display_labels) + assert_array_equal(x_ticks, expected_display_labels_str) + assert_array_equal(y_ticks, expected_display_labels_str) + + image_data = disp.im_.get_array().data + assert_allclose(image_data, cm) + + if include_values: + assert disp.text_.shape == (n_classes, n_classes) + fmt = '.2g' + expected_text = np.array([format(v, fmt) for v in cm.ravel(order="C")]) + text_text = np.array([ + t.get_text() for t in disp.text_.ravel(order="C")]) + assert_array_equal(expected_text, text_text) + else: + assert disp.text_ is None + + +def test_confusion_matrix_display(pyplot, data, fitted_clf, y_pred, n_classes): + X, y = data + + cm = confusion_matrix(y, y_pred) + disp = plot_confusion_matrix(fitted_clf, X, y, normalize=None, + include_values=True, cmap='viridis', + xticks_rotation=45.0) + + assert_allclose(disp.confusion_matrix, cm) + assert disp.text_.shape == (n_classes, n_classes) + + rotations = [tick.get_rotation() for tick in disp.ax_.get_xticklabels()] + assert_allclose(rotations, 45.0) + + image_data = disp.im_.get_array().data + assert_allclose(image_data, cm) + + disp.plot(cmap='plasma') + assert disp.im_.get_cmap().name == 'plasma' + + disp.plot(include_values=False) + assert disp.text_ is None + + disp.plot(xticks_rotation=90.0) + rotations = [tick.get_rotation() for tick in disp.ax_.get_xticklabels()] + assert_allclose(rotations, 90.0) + + disp.plot(values_format='e') + expected_text = np.array([format(v, 'e') for v in cm.ravel(order="C")]) + text_text = np.array([ + t.get_text() for t in disp.text_.ravel(order="C")]) + assert_array_equal(expected_text, text_text) + + +def test_confusion_matrix_contrast(pyplot): + # make sure text color is appropriate depending on background + + cm = np.eye(2) / 2 + disp = ConfusionMatrixDisplay(cm, display_labels=[0, 1]) + + disp.plot(cmap=pyplot.cm.gray) + # diagonal text is black + assert_allclose(disp.text_[0, 0].get_color(), [0.0, 0.0, 0.0, 1.0]) + assert_allclose(disp.text_[1, 1].get_color(), [0.0, 0.0, 0.0, 1.0]) + + # off-diagonal text is white + assert_allclose(disp.text_[0, 1].get_color(), [1.0, 1.0, 1.0, 1.0]) + assert_allclose(disp.text_[1, 0].get_color(), [1.0, 1.0, 1.0, 1.0]) + + disp.plot(cmap=pyplot.cm.gray_r) + # diagonal text is white + assert_allclose(disp.text_[0, 1].get_color(), [0.0, 0.0, 0.0, 1.0]) + assert_allclose(disp.text_[1, 0].get_color(), [0.0, 0.0, 0.0, 1.0]) + + # off-diagonal text is black + assert_allclose(disp.text_[0, 0].get_color(), [1.0, 1.0, 1.0, 1.0]) + assert_allclose(disp.text_[1, 1].get_color(), [1.0, 1.0, 1.0, 1.0]) + + # Regression test for #15920 + cm = np.array([[19, 34], [32, 58]]) + disp = ConfusionMatrixDisplay(cm, display_labels=[0, 1]) + + disp.plot(cmap=pyplot.cm.Blues) + min_color = pyplot.cm.Blues(0) + max_color = pyplot.cm.Blues(255) + assert_allclose(disp.text_[0, 0].get_color(), max_color) + assert_allclose(disp.text_[0, 1].get_color(), max_color) + assert_allclose(disp.text_[1, 0].get_color(), max_color) + assert_allclose(disp.text_[1, 1].get_color(), min_color) + + +@pytest.mark.parametrize( + "clf", [LogisticRegression(), + make_pipeline(StandardScaler(), LogisticRegression()), + make_pipeline(make_column_transformer((StandardScaler(), [0, 1])), + LogisticRegression())]) +def test_confusion_matrix_pipeline(pyplot, clf, data, n_classes): + X, y = data + with pytest.raises(NotFittedError): + plot_confusion_matrix(clf, X, y) + clf.fit(X, y) + y_pred = clf.predict(X) + + disp = plot_confusion_matrix(clf, X, y) + cm = confusion_matrix(y, y_pred) + + assert_allclose(disp.confusion_matrix, cm) + assert disp.text_.shape == (n_classes, n_classes) + + +@pytest.mark.parametrize("values_format", ['e', 'n']) +def test_confusion_matrix_text_format(pyplot, data, y_pred, n_classes, + fitted_clf, values_format): + # Make sure plot text is formatted with 'values_format'. + X, y = data + cm = confusion_matrix(y, y_pred) + disp = plot_confusion_matrix(fitted_clf, X, y, + include_values=True, + values_format=values_format) + + assert disp.text_.shape == (n_classes, n_classes) + + expected_text = np.array([format(v, values_format) + for v in cm.ravel()]) + text_text = np.array([ + t.get_text() for t in disp.text_.ravel()]) + assert_array_equal(expected_text, text_text) + + +def test_confusion_matrix_standard_format(pyplot): + cm = np.array([[10000000, 0], [123456, 12345678]]) + plotted_text = ConfusionMatrixDisplay(cm, [False, True]).plot().text_ + # Values should be shown as whole numbers 'd', + # except the first number which should be shown as 1e+07 (longer length) + # and the last number will be showns as 1.2e+07 (longer length) + test = [t.get_text() for t in plotted_text.ravel()] + assert test == ['1e+07', '0', '123456', '1.2e+07'] + + cm = np.array([[0.1, 10], [100, 0.525]]) + plotted_text = ConfusionMatrixDisplay(cm, [False, True]).plot().text_ + # Values should now formatted as '.2g', since there's a float in + # Values are have two dec places max, (e.g 100 becomes 1e+02) + test = [t.get_text() for t in plotted_text.ravel()] + assert test == ['0.1', '10', '1e+02', '0.53'] diff --git a/sklearn/metrics/_plot/tests/test_plot_precision_recall.py b/sklearn/metrics/_plot/tests/test_plot_precision_recall.py index 03c6778254e18..f22b112e96dc7 100644 --- a/sklearn/metrics/_plot/tests/test_plot_precision_recall.py +++ b/sklearn/metrics/_plot/tests/test_plot_precision_recall.py @@ -7,6 +7,7 @@ from sklearn.metrics import average_precision_score from sklearn.metrics import precision_recall_curve from sklearn.datasets import make_classification +from sklearn.datasets import load_breast_cancer from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor from sklearn.linear_model import LogisticRegression from sklearn.exceptions import NotFittedError @@ -15,6 +16,12 @@ from sklearn.compose import make_column_transformer +# TODO: Remove when https://github.com/numpy/numpy/issues/14397 is resolved +pytestmark = pytest.mark.filterwarnings( + "ignore:In future, it will be an error for 'np.bool_':DeprecationWarning:" + "matplotlib.*") + + def test_errors(pyplot): X, y_multiclass = make_classification(n_classes=3, n_samples=50, n_informative=3, @@ -30,12 +37,12 @@ def test_errors(pyplot): multi_clf = DecisionTreeClassifier().fit(X, y_multiclass) # Fitted multiclass classifier with binary data - msg = "DecisionTreeClassifier should be a binary classifer" + msg = "DecisionTreeClassifier should be a binary classifier" with pytest.raises(ValueError, match=msg): plot_precision_recall_curve(multi_clf, X, y_binary) reg = DecisionTreeRegressor().fit(X, y_multiclass) - msg = "DecisionTreeRegressor should be a binary classifer" + msg = "DecisionTreeRegressor should be a binary classifier" with pytest.raises(ValueError, match=msg): plot_precision_recall_curve(reg, X, y_binary) @@ -126,3 +133,40 @@ def test_precision_recall_curve_pipeline(pyplot, clf): clf.fit(X, y) disp = plot_precision_recall_curve(clf, X, y) assert disp.estimator_name == clf.__class__.__name__ + + +def test_precision_recall_curve_string_labels(pyplot): + # regression test #15738 + cancer = load_breast_cancer() + X = cancer.data + y = cancer.target_names[cancer.target] + + lr = make_pipeline(StandardScaler(), LogisticRegression()) + lr.fit(X, y) + for klass in cancer.target_names: + assert klass in lr.classes_ + disp = plot_precision_recall_curve(lr, X, y) + + y_pred = lr.predict_proba(X)[:, 1] + avg_prec = average_precision_score(y, y_pred, + pos_label=lr.classes_[1]) + + assert disp.average_precision == pytest.approx(avg_prec) + assert disp.estimator_name == lr.__class__.__name__ + + +def test_plot_precision_recall_curve_estimator_name_multiple_calls(pyplot): + # non-regression test checking that the `name` used when calling + # `plot_roc_curve` is used as well when calling `disp.plot()` + X, y = make_classification(n_classes=2, n_samples=50, random_state=0) + clf_name = "my hand-crafted name" + clf = LogisticRegression().fit(X, y) + disp = plot_precision_recall_curve(clf, X, y, name=clf_name) + assert disp.estimator_name == clf_name + pyplot.close("all") + disp.plot() + assert clf_name in disp.line_.get_label() + pyplot.close("all") + clf_name = "another_name" + disp.plot(name=clf_name) + assert clf_name in disp.line_.get_label() diff --git a/sklearn/metrics/_plot/tests/test_plot_roc_curve.py b/sklearn/metrics/_plot/tests/test_plot_roc_curve.py index 609a422c3d13a..699387ff4cfa3 100644 --- a/sklearn/metrics/_plot/tests/test_plot_roc_curve.py +++ b/sklearn/metrics/_plot/tests/test_plot_roc_curve.py @@ -14,6 +14,12 @@ from sklearn.compose import make_column_transformer +# TODO: Remove when https://github.com/numpy/numpy/issues/14397 is resolved +pytestmark = pytest.mark.filterwarnings( + "ignore:In future, it will be an error for 'np.bool_':DeprecationWarning:" + "matplotlib.*") + + @pytest.fixture(scope="module") def data(): return load_iris(return_X_y=True) @@ -30,7 +36,7 @@ def test_plot_roc_curve_error_non_binary(pyplot, data): clf = DecisionTreeClassifier() clf.fit(X, y) - msg = "DecisionTreeClassifier should be a binary classifer" + msg = "DecisionTreeClassifier should be a binary classifier" with pytest.raises(ValueError, match=msg): plot_roc_curve(clf, X, y) @@ -125,4 +131,22 @@ def test_roc_curve_not_fitted_errors(pyplot, data_binary, clf): plot_roc_curve(clf, X, y) clf.fit(X, y) disp = plot_roc_curve(clf, X, y) + assert clf.__class__.__name__ in disp.line_.get_label() assert disp.estimator_name == clf.__class__.__name__ + + +def test_plot_roc_curve_estimator_name_multiple_calls(pyplot, data_binary): + # non-regression test checking that the `name` used when calling + # `plot_roc_curve` is used as well when calling `disp.plot()` + X, y = data_binary + clf_name = "my hand-crafted name" + clf = LogisticRegression().fit(X, y) + disp = plot_roc_curve(clf, X, y, name=clf_name) + assert disp.estimator_name == clf_name + pyplot.close("all") + disp.plot() + assert clf_name in disp.line_.get_label() + pyplot.close("all") + clf_name = "another_name" + disp.plot(name=clf_name) + assert clf_name in disp.line_.get_label() diff --git a/sklearn/metrics/_ranking.py b/sklearn/metrics/_ranking.py index c271781638668..e525539c0d706 100644 --- a/sklearn/metrics/_ranking.py +++ b/sklearn/metrics/_ranking.py @@ -248,27 +248,32 @@ def roc_auc_score(y_true, y_score, average="macro", sample_weight=None, """Compute Area Under the Receiver Operating Characteristic Curve (ROC AUC) from prediction scores. - Note: this implementation is restricted to the binary classification task - or multilabel classification task in label indicator format. + Note: this implementation can be used with binary, multiclass and + multilabel classification, but some restrictions apply (see Parameters). Read more in the :ref:`User Guide `. Parameters ---------- - y_true : array, shape = [n_samples] or [n_samples, n_classes] - True binary labels or binary label indicators. - The multiclass case expects shape = [n_samples] and labels - with values in ``range(n_classes)``. - - y_score : array, shape = [n_samples] or [n_samples, n_classes] - Target scores, can either be probability estimates of the positive - class, confidence values, or non-thresholded measure of decisions - (as returned by "decision_function" on some classifiers). For binary - y_true, y_score is supposed to be the score of the class with greater - label. The multiclass case expects shape = [n_samples, n_classes] - where the scores correspond to probability estimates. - - average : string, [None, 'micro', 'macro' (default), 'samples', 'weighted'] + y_true : array-like of shape (n_samples,) or (n_samples, n_classes) + True labels or binary label indicators. The binary and multiclass cases + expect labels with shape (n_samples,) while the multilabel case expects + binary label indicators with shape (n_samples, n_classes). + + y_score : array-like of shape (n_samples,) or (n_samples, n_classes) + Target scores. In the binary and multilabel cases, these can be either + probability estimates or non-thresholded decision values (as returned + by `decision_function` on some classifiers). In the multiclass case, + these must be probability estimates which sum to 1. The binary + case expects a shape (n_samples,), and the scores must be the scores of + the class with the greater label. The multiclass and multilabel + cases expect a shape (n_samples, n_classes). In the multiclass case, + the order of the class scores must correspond to the order of + ``labels``, if provided, or else to the numerical or lexicographical + order of the labels in ``y_true``. + + average : {'micro', 'macro', 'samples', 'weighted'} or None, \ + default='macro' If ``None``, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data: Note: multiclass ROC AUC currently only handles the 'macro' and @@ -291,26 +296,32 @@ def roc_auc_score(y_true, y_score, average="macro", sample_weight=None, sample_weight : array-like of shape (n_samples,), default=None Sample weights. - max_fpr : float > 0 and <= 1, optional - If not ``None``, the standardized partial AUC [3]_ over the range + max_fpr : float > 0 and <= 1, default=None + If not ``None``, the standardized partial AUC [2]_ over the range [0, max_fpr] is returned. For the multiclass case, ``max_fpr``, should be either equal to ``None`` or ``1.0`` as AUC ROC partial computation currently is not supported for multiclass. - multi_class : string, 'ovr' or 'ovo', optional(default='raise') - Determines the type of multiclass configuration to use. - ``multi_class`` must be provided when ``y_true`` is multiclass. + multi_class : {'raise', 'ovr', 'ovo'}, default='raise' + Multiclass only. Determines the type of configuration to use. The + default value raises an error, so either ``'ovr'`` or ``'ovo'`` must be + passed explicitly. ``'ovr'``: - Calculate metrics for the multiclass case using the one-vs-rest - approach. + Computes the AUC of each class against the rest [3]_ [4]_. This + treats the multiclass case in the same way as the multilabel case. + Sensitive to class imbalance even when ``average == 'macro'``, + because class imbalance affects the composition of each of the + 'rest' groupings. ``'ovo'``: - Calculate metrics for the multiclass case using the one-vs-one - approach. + Computes the average AUC of all possible pairwise combinations of + classes [5]_. Insensitive to class imbalance when + ``average == 'macro'``. - labels : array, shape = [n_classes] or None, optional (default=None) - List of labels to index ``y_score`` used for multiclass. If ``None``, - the lexicon order of ``y_true`` is used to index ``y_score``. + labels : array-like of shape (n_classes,), default=None + Multiclass only. List of labels that index the classes in ``y_score``. + If ``None``, the numerical or lexicographical order of the labels in + ``y_true`` is used. Returns ------- @@ -321,12 +332,22 @@ def roc_auc_score(y_true, y_score, average="macro", sample_weight=None, .. [1] `Wikipedia entry for the Receiver operating characteristic `_ - .. [2] Fawcett T. An introduction to ROC analysis[J]. Pattern Recognition - Letters, 2006, 27(8):861-874. - - .. [3] `Analyzing a portion of the ROC curve. McClish, 1989 + .. [2] `Analyzing a portion of the ROC curve. McClish, 1989 `_ + .. [3] Provost, F., Domingos, P. (2000). Well-trained PETs: Improving + probability estimation trees (Section 6.2), CeDER Working Paper + #IS-00-04, Stern School of Business, New York University. + + .. [4] `Fawcett, T. (2006). An introduction to ROC analysis. Pattern + Recognition Letters, 27(8), 861-874. + `_ + + .. [5] `Hand, D.J., Till, R.J. (2001). A Simple Generalisation of the Area + Under the ROC Curve for Multiple Class Classification Problems. + Machine Learning, 45(2), 171-186. + `_ + See also -------- average_precision_score : Area under the precision-recall curve @@ -341,7 +362,6 @@ def roc_auc_score(y_true, y_score, average="macro", sample_weight=None, >>> y_scores = np.array([0.1, 0.4, 0.35, 0.8]) >>> roc_auc_score(y_true, y_scores) 0.75 - """ y_type = type_of_target(y_true) @@ -525,14 +545,23 @@ def _binary_clf_curve(y_true, y_score, pos_label=None, sample_weight=None): sample_weight = column_or_1d(sample_weight) # ensure binary classification if pos_label is not specified + # classes.dtype.kind in ('O', 'U', 'S') is required to avoid + # triggering a FutureWarning by calling np.array_equal(a, b) + # when elements in the two arrays are not comparable. classes = np.unique(y_true) - if (pos_label is None and - not (np.array_equal(classes, [0, 1]) or - np.array_equal(classes, [-1, 1]) or - np.array_equal(classes, [0]) or - np.array_equal(classes, [-1]) or - np.array_equal(classes, [1]))): - raise ValueError("Data is not binary and pos_label is not specified") + if (pos_label is None and ( + classes.dtype.kind in ('O', 'U', 'S') or + not (np.array_equal(classes, [0, 1]) or + np.array_equal(classes, [-1, 1]) or + np.array_equal(classes, [0]) or + np.array_equal(classes, [-1]) or + np.array_equal(classes, [1])))): + classes_repr = ", ".join(repr(c) for c in classes) + raise ValueError("y_true takes value in {{{classes_repr}}} and " + "pos_label is not specified: either make y_true " + "take value in {{0, 1}} or {{-1, 1}} or " + "pass pos_label explicitly.".format( + classes_repr=classes_repr)) elif pos_label is None: pos_label = 1. diff --git a/sklearn/metrics/_regression.py b/sklearn/metrics/_regression.py index d0226e62bb7ec..6026a5293806a 100644 --- a/sklearn/metrics/_regression.py +++ b/sklearn/metrics/_regression.py @@ -22,11 +22,10 @@ # Christian Lorentzen # License: BSD 3 clause - import numpy as np -from scipy.special import xlogy import warnings +from .._loss.glm_distribution import TweedieDistribution from ..utils.validation import (check_array, check_consistent_length, _num_samples) from ..utils.validation import column_or_1d @@ -255,7 +254,7 @@ def mean_squared_error(y_true, y_pred, weights=sample_weight) if isinstance(multioutput, str): if multioutput == 'raw_values': - return output_errors + return output_errors if squared else np.sqrt(output_errors) elif multioutput == 'uniform_average': # pass None as weights to np.average: uniform mean multioutput = None @@ -669,7 +668,7 @@ def mean_tweedie_deviance(y_true, y_pred, sample_weight=None, power=0): y_pred : array-like of shape (n_samples,) Estimated target values. - sample_weight : array-like, shape (n_samples,), optional + sample_weight : array-like of shape (n_samples,), default=None Sample weights. power : float, default=0 @@ -714,47 +713,8 @@ def mean_tweedie_deviance(y_true, y_pred, sample_weight=None, power=0): sample_weight = column_or_1d(sample_weight) sample_weight = sample_weight[:, np.newaxis] - message = ("Mean Tweedie deviance error with power={} can only be used on " - .format(power)) - if power < 0: - # 'Extreme stable', y_true any realy number, y_pred > 0 - if (y_pred <= 0).any(): - raise ValueError(message + "strictly positive y_pred.") - dev = 2 * (np.power(np.maximum(y_true, 0), 2 - power) - / ((1 - power) * (2 - power)) - - y_true * np.power(y_pred, 1 - power)/(1 - power) - + np.power(y_pred, 2 - power)/(2 - power)) - elif power == 0: - # Normal distribution, y_true and y_pred any real number - dev = (y_true - y_pred)**2 - elif power < 1: - raise ValueError("Tweedie deviance is only defined for power<=0 and " - "power>=1.") - elif power == 1: - # Poisson distribution, y_true >= 0, y_pred > 0 - if (y_true < 0).any() or (y_pred <= 0).any(): - raise ValueError(message + "non-negative y_true and strictly " - "positive y_pred.") - dev = 2 * (xlogy(y_true, y_true/y_pred) - y_true + y_pred) - elif power == 2: - # Gamma distribution, y_true and y_pred > 0 - if (y_true <= 0).any() or (y_pred <= 0).any(): - raise ValueError(message + "strictly positive y_true and y_pred.") - dev = 2 * (np.log(y_pred/y_true) + y_true/y_pred - 1) - else: - if power < 2: - # 1 < p < 2 is Compound Poisson, y_true >= 0, y_pred > 0 - if (y_true < 0).any() or (y_pred <= 0).any(): - raise ValueError(message + "non-negative y_true and strictly " - "positive y_pred.") - else: - if (y_true <= 0).any() or (y_pred <= 0).any(): - raise ValueError(message + "strictly positive y_true and " - "y_pred.") - - dev = 2 * (np.power(y_true, 2 - power)/((1 - power) * (2 - power)) - - y_true * np.power(y_pred, 1 - power)/(1 - power) - + np.power(y_pred, 2 - power)/(2 - power)) + dist = TweedieDistribution(power=power) + dev = dist.unit_deviance(y_true, y_pred, check_input=True) return np.average(dev, weights=sample_weight) @@ -763,7 +723,7 @@ def mean_poisson_deviance(y_true, y_pred, sample_weight=None): """Mean Poisson deviance regression loss. Poisson deviance is equivalent to the Tweedie deviance with - the power parameter `p=1`. + the power parameter `power=1`. Read more in the :ref:`User Guide `. @@ -775,7 +735,7 @@ def mean_poisson_deviance(y_true, y_pred, sample_weight=None): y_pred : array-like of shape (n_samples,) Estimated target values. Requires y_pred > 0. - sample_weight : array-like, shape (n_samples,), optional + sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns @@ -800,8 +760,8 @@ def mean_gamma_deviance(y_true, y_pred, sample_weight=None): """Mean Gamma deviance regression loss. Gamma deviance is equivalent to the Tweedie deviance with - the power parameter `p=2`. It is invariant to scaling of - the target variable, and mesures relative errors. + the power parameter `power=2`. It is invariant to scaling of + the target variable, and measures relative errors. Read more in the :ref:`User Guide `. @@ -813,7 +773,7 @@ def mean_gamma_deviance(y_true, y_pred, sample_weight=None): y_pred : array-like of shape (n_samples,) Estimated target values. Requires y_pred > 0. - sample_weight : array-like, shape (n_samples,), optional + sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns diff --git a/sklearn/metrics/cluster/__init__.py b/sklearn/metrics/cluster/__init__.py index 9d4b2ac87a974..b45c1a8f21774 100644 --- a/sklearn/metrics/cluster/__init__.py +++ b/sklearn/metrics/cluster/__init__.py @@ -20,7 +20,6 @@ from ._unsupervised import silhouette_samples from ._unsupervised import silhouette_score from ._unsupervised import calinski_harabasz_score -from ._unsupervised import calinski_harabaz_score from ._unsupervised import davies_bouldin_score from ._bicluster import consensus_score @@ -29,6 +28,5 @@ "expected_mutual_information", "homogeneity_completeness_v_measure", "homogeneity_score", "mutual_info_score", "v_measure_score", "fowlkes_mallows_score", "entropy", "silhouette_samples", - "silhouette_score", "calinski_harabaz_score", - "calinski_harabasz_score", "davies_bouldin_score", - "consensus_score"] + "silhouette_score", "calinski_harabasz_score", + "davies_bouldin_score", "consensus_score"] diff --git a/sklearn/metrics/cluster/_supervised.py b/sklearn/metrics/cluster/_supervised.py index 146a7338b97fa..dde32dd3f25ce 100644 --- a/sklearn/metrics/cluster/_supervised.py +++ b/sklearn/metrics/cluster/_supervised.py @@ -645,7 +645,7 @@ def mutual_info_score(labels_true, labels_pred, contingency=None): log_outer = -np.log(outer) + log(pi.sum()) + log(pj.sum()) mi = (contingency_nm * (log_contingency_nm - log(contingency_sum)) + contingency_nm * log_outer) - return mi.sum() + return np.clip(mi.sum(), 0.0, None) def adjusted_mutual_info_score(labels_true, labels_pred, diff --git a/sklearn/metrics/cluster/_unsupervised.py b/sklearn/metrics/cluster/_unsupervised.py index f341f3e80b5c8..a0eaa5e84240a 100644 --- a/sklearn/metrics/cluster/_unsupervised.py +++ b/sklearn/metrics/cluster/_unsupervised.py @@ -78,11 +78,10 @@ def silhouette_score(X, labels, metric='euclidean', sample_size=None, If ``sample_size is None``, no sampling is used. random_state : int, RandomState instance or None, optional (default=None) - The generator used to randomly select a subset of samples. If int, - random_state is the seed used by the random number generator; If - RandomState instance, random_state is the random number generator; If - None, the random number generator is the RandomState instance used by - `np.random`. Used when ``sample_size is not None``. + Determines random number generation for selecting a subset of samples. + Used when ``sample_size is not None``. + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary `. **kwds : optional keyword parameters Any further parameters are passed directly to the distance function. @@ -299,13 +298,6 @@ def calinski_harabasz_score(X, labels): (intra_disp * (n_labels - 1.))) -@deprecated("Function 'calinski_harabaz_score' has been renamed to " - "'calinski_harabasz_score' " - "and will be removed in version 0.23.") -def calinski_harabaz_score(X, labels): - return calinski_harabasz_score(X, labels) - - def davies_bouldin_score(X, labels): """Computes the Davies-Bouldin score. diff --git a/sklearn/metrics/cluster/tests/test_supervised.py b/sklearn/metrics/cluster/tests/test_supervised.py index d2d2dd6faaea4..65b585894d075 100644 --- a/sklearn/metrics/cluster/tests/test_supervised.py +++ b/sklearn/metrics/cluster/tests/test_supervised.py @@ -337,3 +337,14 @@ def test_fowlkes_mallows_score_properties(): # symmetric and permutation(both together) score_both = fowlkes_mallows_score(labels_b, (labels_a + 2) % 3) assert_almost_equal(score_both, expected) + + +@pytest.mark.parametrize('labels_true, labels_pred', [ + (['a'] * 6, [1, 1, 0, 0, 1, 1]), + ([1] * 6, [1, 1, 0, 0, 1, 1]), + ([1, 1, 0, 0, 1, 1], ['a'] * 6), + ([1, 1, 0, 0, 1, 1], [1] * 6), +]) +def test_mutual_info_score_positive_constant_label(labels_true, labels_pred): + # non-regression test for #16355 + assert mutual_info_score(labels_true, labels_pred) >= 0 diff --git a/sklearn/metrics/cluster/tests/test_unsupervised.py b/sklearn/metrics/cluster/tests/test_unsupervised.py index 6a00f771273cb..f169a9242daf0 100644 --- a/sklearn/metrics/cluster/tests/test_unsupervised.py +++ b/sklearn/metrics/cluster/tests/test_unsupervised.py @@ -10,7 +10,6 @@ from sklearn.metrics.cluster import silhouette_samples from sklearn.metrics import pairwise_distances from sklearn.metrics.cluster import calinski_harabasz_score -from sklearn.metrics.cluster import calinski_harabaz_score from sklearn.metrics.cluster import davies_bouldin_score @@ -221,15 +220,6 @@ def test_calinski_harabasz_score(): 45 * (40 - 4) / (5 * (4 - 1))) -def test_deprecated_calinski_harabaz_score(): - depr_message = ("Function 'calinski_harabaz_score' has been renamed " - "to 'calinski_harabasz_score' " - "and will be removed in version 0.23.") - assert_warns_message(FutureWarning, depr_message, - calinski_harabaz_score, - np.ones((10, 2)), [0] * 5 + [1] * 5) - - def test_davies_bouldin_score(): assert_raises_on_only_one_label(davies_bouldin_score) assert_raises_on_all_points_same_cluster(davies_bouldin_score) diff --git a/sklearn/metrics/pairwise.py b/sklearn/metrics/pairwise.py index b3ebec934e3a0..25646acb49ea7 100644 --- a/sklearn/metrics/pairwise.py +++ b/sklearn/metrics/pairwise.py @@ -406,20 +406,25 @@ def nan_euclidean_distances(X, Y=None, squared=False, distances -= np.dot(XX, missing_Y.T) distances -= np.dot(missing_X, YY.T) - present_coords_cnt = np.dot(1 - missing_X, 1 - missing_Y.T) - present_mask = (present_coords_cnt != 0) - distances[present_mask] *= (X.shape[1] / present_coords_cnt[present_mask]) + np.clip(distances, 0, None, out=distances) if X is Y: # Ensure that distances between vectors and themselves are set to 0.0. # This may not be the case due to floating point rounding errors. np.fill_diagonal(distances, 0.0) + present_X = 1 - missing_X + present_Y = present_X if Y is X else ~missing_Y + present_count = np.dot(present_X, present_Y.T) + distances[present_count == 0] = np.nan + # avoid divide by zero + np.maximum(1, present_count, out=present_count) + distances /= present_count + distances *= X.shape[1] + if not squared: np.sqrt(distances, out=distances) - # coordinates with no common coordinates have a nan distance - distances[~present_mask] = np.nan return distances @@ -1403,6 +1408,8 @@ def _pairwise_callable(X, Y, metric, force_all_finite=True, **kwds): def _check_chunk_size(reduced, chunk_size): """Checks chunk is a sequence of expected size or a tuple of same """ + if reduced is None: + return is_tuple = isinstance(reduced, tuple) if not is_tuple: reduced = (reduced,) @@ -1463,8 +1470,9 @@ def pairwise_distances_chunked(X, Y=None, reduce_func=None, reducing it to needed values. ``reduce_func(D_chunk, start)`` is called repeatedly, where ``D_chunk`` is a contiguous vertical slice of the pairwise distance matrix, starting at row ``start``. - It should return an array, a list, or a sparse matrix of length - ``D_chunk.shape[0]``, or a tuple of such objects. + It should return one of: None; an array, a list, or a sparse matrix + of length ``D_chunk.shape[0]``; or a tuple of such objects. Returning + None is useful for in-place operations, rather than reductions. If None, pairwise_distances_chunked returns a generator of vertical chunks of the distance matrix. diff --git a/sklearn/metrics/tests/test_classification.py b/sklearn/metrics/tests/test_classification.py index 6d981ee4da53c..be6364e63b2cd 100644 --- a/sklearn/metrics/tests/test_classification.py +++ b/sklearn/metrics/tests/test_classification.py @@ -1,6 +1,8 @@ from functools import partial from itertools import product +from itertools import chain +from itertools import permutations import warnings import re @@ -17,7 +19,7 @@ from sklearn.utils._testing import assert_almost_equal from sklearn.utils._testing import assert_array_equal from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_warns +from sklearn.utils._testing import assert_allclose from sklearn.utils._testing import assert_warns_div0 from sklearn.utils._testing import assert_no_warnings from sklearn.utils._testing import assert_warns_message @@ -35,7 +37,6 @@ from sklearn.metrics import hamming_loss from sklearn.metrics import hinge_loss from sklearn.metrics import jaccard_score -from sklearn.metrics import jaccard_similarity_score from sklearn.metrics import log_loss from sklearn.metrics import matthews_corrcoef from sklearn.metrics import precision_recall_fscore_support @@ -152,6 +153,22 @@ def test_classification_report_dictionary_output(): assert type(expected_report['macro avg']['support']) == int +@pytest.mark.parametrize('zero_division', ["warn", 0, 1]) +def test_classification_report_zero_division_warning(zero_division): + y_true, y_pred = ["a", "b", "c"], ["a", "b", "d"] + with warnings.catch_warnings(record=True) as record: + classification_report( + y_true, y_pred, zero_division=zero_division, output_dict=True) + if zero_division == "warn": + assert len(record) > 1 + for item in record: + msg = ("Use `zero_division` parameter to control this " + "behavior.") + assert msg in str(item.message) + else: + assert not record + + def test_multilabel_accuracy_score_subset_accuracy(): # Dense label indicator matrix format y1 = np.array([[0, 1, 1], [1, 0, 1]]) @@ -484,7 +501,7 @@ def test_multilabel_confusion_matrix_errors(): # Bad sample_weight with pytest.raises(ValueError, match="inconsistent numbers of samples"): multilabel_confusion_matrix(y_true, y_pred, sample_weight=[1, 2]) - with pytest.raises(ValueError, match="bad input shape"): + with pytest.raises(ValueError, match="should be a 1d array"): multilabel_confusion_matrix(y_true, y_pred, sample_weight=[[1, 2, 3], [2, 3, 4], @@ -509,6 +526,46 @@ def test_multilabel_confusion_matrix_errors(): [[1, 2, 0], [1, 0, 2]]) +@pytest.mark.parametrize( + "normalize, cm_dtype, expected_results", + [('true', 'f', 0.333333333), + ('pred', 'f', 0.333333333), + ('all', 'f', 0.1111111111), + (None, 'i', 2)] +) +def test_confusion_matrix_normalize(normalize, cm_dtype, expected_results): + y_test = [0, 1, 2] * 6 + y_pred = list(chain(*permutations([0, 1, 2]))) + cm = confusion_matrix(y_test, y_pred, normalize=normalize) + assert_allclose(cm, expected_results) + assert cm.dtype.kind == cm_dtype + + +def test_confusion_matrix_normalize_wrong_option(): + y_test = [0, 0, 0, 0, 1, 1, 1, 1] + y_pred = [0, 0, 0, 0, 0, 0, 0, 0] + with pytest.raises(ValueError, match='normalize must be one of'): + confusion_matrix(y_test, y_pred, normalize=True) + + +def test_confusion_matrix_normalize_single_class(): + y_test = [0, 0, 0, 0, 1, 1, 1, 1] + y_pred = [0, 0, 0, 0, 0, 0, 0, 0] + + cm_true = confusion_matrix(y_test, y_pred, normalize='true') + assert cm_true.sum() == pytest.approx(2.0) + + # additionally check that no warnings are raised due to a division by zero + with pytest.warns(None) as rec: + cm_pred = confusion_matrix(y_test, y_pred, normalize='pred') + assert not rec + assert cm_pred.sum() == pytest.approx(1.0) + + with pytest.warns(None) as rec: + cm_pred = confusion_matrix(y_pred, y_test, normalize='true') + assert not rec + + def test_cohen_kappa(): # These label vectors reproduce the contingency matrix from Artstein and # Poesio (2008), Table 1: np.array([[20, 20], [10, 50]]). @@ -851,10 +908,28 @@ def test_confusion_matrix_multiclass_subset_labels(): assert_array_equal(cm, [[18, 0], [0, 0]]) - # check for exception when none of the specified labels are in y_true - with pytest.raises(ValueError): - confusion_matrix(y_true, y_pred, - labels=[extra_label, extra_label + 1]) + +@pytest.mark.parametrize( + "labels, err_msg", + [([], "'labels' should contains at least one label."), + ([3, 4], "At least one label specified must be in y_true")], + ids=["empty list", "unknown labels"] +) +def test_confusion_matrix_error(labels, err_msg): + y_true, y_pred, _ = make_prediction(binary=False) + with pytest.raises(ValueError, match=err_msg): + confusion_matrix(y_true, y_pred, labels=labels) + + +@pytest.mark.parametrize( + 'labels', (None, [0, 1], [0, 1, 2]), + ids=['None', 'binary', 'multiclass'] +) +def test_confusion_matrix_on_zero_length_input(labels): + expected_n_classes = len(labels) if labels else 0 + expected = np.zeros((expected_n_classes, expected_n_classes), dtype=np.int) + cm = confusion_matrix([], [], labels=labels) + assert_array_equal(cm, expected) def test_confusion_matrix_dtype(): @@ -1141,11 +1216,6 @@ def test_multilabel_hamming_loss(): assert hamming_loss(y1, np.zeros_like(y1), sample_weight=w) == 2. / 3 # sp_hamming only works with 1-D arrays assert hamming_loss(y1[0], y2[0]) == sp_hamming(y1[0], y2[0]) - assert_warns_message(FutureWarning, - "The labels parameter is unused. It was" - " deprecated in version 0.21 and" - " will be removed in version 0.23", - hamming_loss, y1, y2, labels=[0, 1]) def test_jaccard_score_validation(): @@ -2202,22 +2272,3 @@ def test_balanced_accuracy_score(y_true, y_pred): adjusted = balanced_accuracy_score(y_true, y_pred, adjusted=True) chance = balanced_accuracy_score(y_true, np.full_like(y_true, y_true[0])) assert adjusted == (balanced - chance) / (1 - chance) - - -def test_multilabel_jaccard_similarity_score_deprecation(): - # Dense label indicator matrix format - y1 = np.array([[0, 1, 1], [1, 0, 1]]) - y2 = np.array([[0, 0, 1], [1, 0, 1]]) - - # size(y1 \inter y2) = [1, 2] - # size(y1 \union y2) = [2, 2] - - jss = partial(assert_warns, FutureWarning, - jaccard_similarity_score) - assert jss(y1, y2) == 0.75 - assert jss(y1, y1) == 1 - assert jss(y2, y2) == 1 - assert jss(y2, np.logical_not(y2)) == 0 - assert jss(y1, np.logical_not(y1)) == 0 - assert jss(y1, np.zeros(y1.shape)) == 0 - assert jss(y2, np.zeros(y1.shape)) == 0 diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py index 5f93810f0b407..7301d21a35f39 100644 --- a/sklearn/metrics/tests/test_common.py +++ b/sklearn/metrics/tests/test_common.py @@ -115,7 +115,7 @@ "unnormalized_accuracy_score": partial(accuracy_score, normalize=False), # `confusion_matrix` returns absolute values and hence behaves unnormalized - # . Naming it with an unnormalized_ prefix is neccessary for this module to + # . Naming it with an unnormalized_ prefix is necessary for this module to # skip sample_weight scaling checks which will fail for unnormalized # metrics. "unnormalized_confusion_matrix": confusion_matrix, @@ -351,8 +351,6 @@ def precision_recall_curve_padded_thresholds(*args, **kwargs): "roc_curve", "precision_recall_curve", - "hamming_loss", - "precision_score", "recall_score", "f1_score", "f2_score", "f0.5_score", "jaccard_score", @@ -1170,8 +1168,7 @@ def check_sample_weight_invariance(name, metric, y1, y2): assert_allclose(unweighted_score, weighted_score) raise ValueError("Unweighted and weighted scores are unexpectedly " "almost equal (%s) and (%s) " - "for %s" % (unweighted_score, - weighted_score, name)) + "for %s" % (unweighted_score, weighted_score, name)) # check that sample_weight can be a list weighted_score_list = metric(y1, y2, diff --git a/sklearn/metrics/tests/test_pairwise.py b/sklearn/metrics/tests/test_pairwise.py index 21898136cb8f9..fdff2c4c3959e 100644 --- a/sklearn/metrics/tests/test_pairwise.py +++ b/sklearn/metrics/tests/test_pairwise.py @@ -483,6 +483,19 @@ def test_pairwise_distances_chunked_reduce(): assert_allclose(np.vstack(S_chunks), S, atol=1e-7) +def test_pairwise_distances_chunked_reduce_none(): + # check that the reduce func is allowed to return None + rng = np.random.RandomState(0) + X = rng.random_sample((10, 4)) + S_chunks = pairwise_distances_chunked(X, None, + reduce_func=lambda dist, start: None, + working_memory=2 ** -16) + assert isinstance(S_chunks, GeneratorType) + S_chunks = list(S_chunks) + assert len(S_chunks) > 1 + assert all(chunk is None for chunk in S_chunks) + + @pytest.mark.parametrize('good_reduce', [ lambda D, start: list(D), lambda D, start: np.array(D), @@ -871,6 +884,23 @@ def test_nan_euclidean_distances_not_trival(missing_value): assert_allclose(D6, D7) +@pytest.mark.parametrize("missing_value", [np.nan, -1]) +def test_nan_euclidean_distances_one_feature_match_positive(missing_value): + # First feature is the only feature that is non-nan and in both + # samples. The result of `nan_euclidean_distances` with squared=True + # should be non-negative. The non-squared version should all be close to 0. + X = np.array([[-122.27, 648., missing_value, 37.85], + [-122.27, missing_value, 2.34701493, missing_value]]) + + dist_squared = nan_euclidean_distances(X, missing_values=missing_value, + squared=True) + assert np.all(dist_squared >= 0) + + dist = nan_euclidean_distances(X, missing_values=missing_value, + squared=False) + assert_allclose(dist, 0.0) + + def test_cosine_distances(): # Check the pairwise Cosine distances computation rng = np.random.RandomState(1337) diff --git a/sklearn/metrics/tests/test_ranking.py b/sklearn/metrics/tests/test_ranking.py index 0275a26055915..4542b8e2a2964 100644 --- a/sklearn/metrics/tests/test_ranking.py +++ b/sklearn/metrics/tests/test_ranking.py @@ -554,7 +554,7 @@ def test_multiclass_ovr_roc_auc_toydata(y_true, labels): result_unweighted) # Tests the weighted, one-vs-rest multiclass ROC AUC algorithm - # on the same input (Provost & Domingos, 2001) + # on the same input (Provost & Domingos, 2000) result_weighted = out_0 * 0.25 + out_1 * 0.25 + out_2 * 0.5 assert_almost_equal( roc_auc_score( @@ -662,14 +662,53 @@ def test_auc_score_non_binary_class(): roc_auc_score(y_true, y_pred) -def test_binary_clf_curve(): +def test_binary_clf_curve_multiclass_error(): rng = check_random_state(404) y_true = rng.randint(0, 3, size=10) y_pred = rng.rand(10) msg = "multiclass format is not supported" + with pytest.raises(ValueError, match=msg): precision_recall_curve(y_true, y_pred) + with pytest.raises(ValueError, match=msg): + roc_curve(y_true, y_pred) + + +@pytest.mark.parametrize("curve_func", [ + precision_recall_curve, + roc_curve, +]) +def test_binary_clf_curve_implicit_pos_label(curve_func): + # Check that using string class labels raises an informative + # error for any supported string dtype: + msg = ("y_true takes value in {'a', 'b'} and pos_label is " + "not specified: either make y_true take " + "value in {0, 1} or {-1, 1} or pass pos_label " + "explicitly.") + with pytest.raises(ValueError, match=msg): + roc_curve(np.array(["a", "b"], dtype='`. warm_start : bool, default to False. If 'warm_start' is True, the solution of the last fitting is used as diff --git a/sklearn/mixture/_gaussian_mixture.py b/sklearn/mixture/_gaussian_mixture.py index 8603115fd202f..1c563984ba00b 100644 --- a/sklearn/mixture/_gaussian_mixture.py +++ b/sklearn/mixture/_gaussian_mixture.py @@ -502,10 +502,12 @@ class GaussianMixture(BaseMixture): (n_components, n_features, n_features) if 'full' random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + Controls the random seed given to the method chosen to initialize the + parameters (see `init_params`). + In addition, it controls the generation of random samples from the + fitted distribution (see the method `sample`). + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. warm_start : bool, default to False. If 'warm_start' is True, the solution of the last fitting is used as diff --git a/sklearn/mixture/tests/test_gaussian_mixture.py b/sklearn/mixture/tests/test_gaussian_mixture.py index ac0a829d2bc24..1282a752271a8 100644 --- a/sklearn/mixture/tests/test_gaussian_mixture.py +++ b/sklearn/mixture/tests/test_gaussian_mixture.py @@ -914,12 +914,8 @@ def test_monotonic_likelihood(): # training log likelihood increases after each iteration. for _ in range(600): prev_log_likelihood = current_log_likelihood - try: - current_log_likelihood = gmm.fit(X).score(X) - except ConvergenceWarning: - pass - assert (current_log_likelihood >= - prev_log_likelihood) + current_log_likelihood = gmm.fit(X).score(X) + assert current_log_likelihood >= prev_log_likelihood if gmm.converged_: break diff --git a/sklearn/model_selection/_search.py b/sklearn/model_selection/_search.py index 4c9b082d355fd..55e770d701858 100644 --- a/sklearn/model_selection/_search.py +++ b/sklearn/model_selection/_search.py @@ -33,11 +33,11 @@ from ..utils import check_random_state from ..utils.fixes import MaskedArray from ..utils.random import sample_without_replacement -from ..utils.validation import indexable, check_is_fitted +from ..utils.validation import indexable, check_is_fitted, _check_fit_params from ..utils.metaestimators import if_delegate_has_method from ..metrics._scorer import _check_multimetric_scoring from ..metrics import check_scoring - +from ..utils import deprecated __all__ = ['GridSearchCV', 'ParameterGrid', 'fit_grid_point', 'ParameterSampler', 'RandomizedSearchCV'] @@ -53,7 +53,7 @@ class ParameterGrid: Parameters ---------- - param_grid : dict of string to sequence, or sequence of such + param_grid : dict of str to sequence, or sequence of such The parameter grid to explore, as a dictionary mapping estimator parameters to sequences of allowed values. @@ -115,7 +115,7 @@ def __iter__(self): Returns ------- - params : iterator over dict of string to any + params : iterator over dict of str to any Yields dictionaries mapping each estimator parameter to one of its allowed values. """ @@ -147,7 +147,7 @@ def __getitem__(self, ind): Returns ------- - params : dict of string to any + params : dict of str to any Equal to list(self)[ind] """ # This is used to make discrete sampling without replacement memory @@ -194,7 +194,7 @@ class ParameterSampler: Parameters ---------- param_distributions : dict - Dictionary with parameters names (string) as keys and distributions + Dictionary with parameters names (`str`) as keys and distributions or lists of parameters to try. Distributions must provide a ``rvs`` method for sampling (such as those from scipy.stats.distributions). If a list is given, it is sampled uniformly. @@ -204,17 +204,16 @@ class ParameterSampler: n_iter : integer Number of parameter settings that are produced. - random_state : int, RandomState instance or None, optional (default=None) + random_state : int or RandomState instance, default=None Pseudo random number generator state used for random uniform sampling from lists of possible values instead of scipy.stats distributions. - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + Pass an int for reproducible output across multiple + function calls. + See :term:`Glossary `. Returns ------- - params : dict of string to any + params : dict of str to any **Yields** dictionaries mapping each estimator parameter to as sampled value. @@ -302,6 +301,11 @@ def __len__(self): return self.n_iter +# FIXME Remove fit_grid_point in 0.25 +@deprecated( + "fit_grid_point is deprecated in version 0.23 " + "and will be removed in version 0.25" +) def fit_grid_point(X, y, estimator, parameters, train, test, scorer, verbose, error_score=np.nan, **fit_params): """Run fit on one set of parameters. @@ -341,11 +345,11 @@ def fit_grid_point(X, y, estimator, parameters, train, test, scorer, **fit_params : kwargs Additional parameter passed to the fit function of the estimator. - error_score : 'raise' or numeric + error_score : 'raise' or numeric, default=np.nan Value to assign to the score if an error occurs in estimator fitting. If set to 'raise', the error is raised. If a numeric value is given, FitFailedWarning is raised. This parameter does not affect the refit - step, which will always raise the error. Default is ``np.nan``. + step, which will always raise the error. Returns ------- @@ -431,7 +435,8 @@ def score(self, X, y=None): Input data, where n_samples is the number of samples and n_features is the number of features. - y : array-like of shape (n_samples, n_output) or (n_samples,), optional + y : array-like of shape (n_samples, n_output) \ + or (n_samples,), default=None Target relative to X for classification or regression; None for unsupervised learning. @@ -561,6 +566,20 @@ def inverse_transform(self, Xt): self._check_is_fitted('inverse_transform') return self.best_estimator_.inverse_transform(Xt) + @property + def n_features_in_(self): + # For consistency with other estimators we raise a AttributeError so + # that hasattr() fails if the search estimator isn't fitted. + try: + check_is_fitted(self) + except NotFittedError as nfe: + raise AttributeError( + "{} object has no n_features_in_ attribute." + .format(self.__class__.__name__) + ) from nfe + + return self.best_estimator_.n_features_in_ + @property def classes_(self): self._check_is_fitted("classes_") @@ -610,16 +629,17 @@ def fit(self, X, y=None, groups=None, **fit_params): Training vector, where n_samples is the number of samples and n_features is the number of features. - y : array-like of shape (n_samples, n_output) or (n_samples,), optional + y : array-like of shape (n_samples, n_output) \ + or (n_samples,), default=None Target relative to X for classification or regression; None for unsupervised learning. - groups : array-like, with shape (n_samples,), optional + groups : array-like of shape (n_samples,), default=None Group labels for the samples used while splitting the dataset into train/test set. Only used in conjunction with a "Group" :term:`cv` instance (e.g., :class:`~sklearn.model_selection.GroupKFold`). - **fit_params : dict of string -> object + **fit_params : dict of str -> object Parameters passed to the ``fit`` method of the estimator """ estimator = self.estimator @@ -648,9 +668,7 @@ def fit(self, X, y=None, groups=None, **fit_params): refit_metric = 'score' X, y, groups = indexable(X, y, groups) - # make sure fit_params are sliceable - fit_params_values = indexable(*fit_params.values()) - fit_params = dict(zip(fit_params.keys(), fit_params_values)) + fit_params = _check_fit_params(X, fit_params) n_splits = cv.get_n_splits(X, y, groups) @@ -863,14 +881,14 @@ class GridSearchCV(BaseSearchCV): or ``scoring`` must be passed. param_grid : dict or list of dictionaries - Dictionary with parameters names (string) as keys and lists of + Dictionary with parameters names (`str`) as keys and lists of parameter settings to try as values, or a list of such dictionaries, in which case the grids spanned by each dictionary in the list are explored. This enables searching over any sequence of parameter settings. - scoring : string, callable, list/tuple, dict or None, default: None - A single string (see :ref:`scoring_parameter`) or a callable + scoring : str, callable, list/tuple or dict, default=None + A single str (see :ref:`scoring_parameter`) or a callable (see :ref:`scoring`) to evaluate the predictions on the test set. For evaluating multiple metrics, either give a list of (unique) strings @@ -884,13 +902,13 @@ class GridSearchCV(BaseSearchCV): If None, the estimator's score method is used. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None Number of jobs to run in parallel. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - pre_dispatch : int, or string, optional + pre_dispatch : int, or str, default=n_jobs Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched @@ -904,10 +922,10 @@ class GridSearchCV(BaseSearchCV): - An int, giving the exact number of total jobs that are spawned - - A string, giving an expression as a function of n_jobs, + - A str, giving an expression as a function of n_jobs, as in '2*n_jobs' - iid : boolean, default=False + iid : bool, default=False If True, return the average score across folds, weighted by the number of samples in each test set. In this case, the data is assumed to be identically distributed across the folds, and the loss minimized is @@ -916,7 +934,7 @@ class GridSearchCV(BaseSearchCV): .. deprecated:: 0.22 Parameter ``iid`` is deprecated in 0.22 and will be removed in 0.24 - cv : int, cross-validation generator or an iterable, optional + cv : int, cross-validation generator or an iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: @@ -935,20 +953,20 @@ class GridSearchCV(BaseSearchCV): .. versionchanged:: 0.22 ``cv`` default value if None changed from 3-fold to 5-fold. - refit : boolean, string, or callable, default=True + refit : bool, str, or callable, default=True Refit an estimator using the best found parameters on the whole dataset. - For multiple metric evaluation, this needs to be a string denoting the + For multiple metric evaluation, this needs to be a `str` denoting the scorer that would be used to find the best parameters for refitting the estimator at the end. Where there are considerations other than maximum score in choosing a best estimator, ``refit`` can be set to a function which returns the selected ``best_index_`` given ``cv_results_``. In that - case, the ``best_estimator_`` and ``best_parameters_`` will be set + case, the ``best_estimator_`` and ``best_params_`` will be set according to the returned ``best_index_`` while the ``best_score_`` - attribute will not be availble. + attribute will not be available. The refitted estimator is made available at the ``best_estimator_`` attribute and permits using ``predict`` directly on this @@ -968,13 +986,13 @@ class GridSearchCV(BaseSearchCV): verbose : integer Controls the verbosity: the higher, the more messages. - error_score : 'raise' or numeric + error_score : 'raise' or numeric, default=np.nan Value to assign to the score if an error occurs in estimator fitting. If set to 'raise', the error is raised. If a numeric value is given, FitFailedWarning is raised. This parameter does not affect the refit - step, which will always raise the error. Default is ``np.nan``. + step, which will always raise the error. - return_train_score : boolean, default=False + return_train_score : bool, default=False If ``False``, the ``cv_results_`` attribute will not include training scores. Computing training scores is used to get insights on how different @@ -1188,7 +1206,7 @@ class RandomizedSearchCV(BaseSearchCV): or ``scoring`` must be passed. param_distributions : dict or list of dicts - Dictionary with parameters names (string) as keys and distributions + Dictionary with parameters names (`str`) as keys and distributions or lists of parameters to try. Distributions must provide a ``rvs`` method for sampling (such as those from scipy.stats.distributions). If a list is given, it is sampled uniformly. @@ -1199,8 +1217,8 @@ class RandomizedSearchCV(BaseSearchCV): Number of parameter settings that are sampled. n_iter trades off runtime vs quality of the solution. - scoring : string, callable, list/tuple, dict or None, default: None - A single string (see :ref:`scoring_parameter`) or a callable + scoring : str, callable, list/tuple or dict, default=None + A single str (see :ref:`scoring_parameter`) or a callable (see :ref:`scoring`) to evaluate the predictions on the test set. For evaluating multiple metrics, either give a list of (unique) strings @@ -1214,13 +1232,13 @@ class RandomizedSearchCV(BaseSearchCV): If None, the estimator's score method is used. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None Number of jobs to run in parallel. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - pre_dispatch : int, or string, optional + pre_dispatch : int, or str, default=None Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched @@ -1234,10 +1252,10 @@ class RandomizedSearchCV(BaseSearchCV): - An int, giving the exact number of total jobs that are spawned - - A string, giving an expression as a function of n_jobs, + - A str, giving an expression as a function of n_jobs, as in '2*n_jobs' - iid : boolean, default=False + iid : bool, default=False If True, return the average score across folds, weighted by the number of samples in each test set. In this case, the data is assumed to be identically distributed across the folds, and the loss minimized is @@ -1246,10 +1264,9 @@ class RandomizedSearchCV(BaseSearchCV): .. deprecated:: 0.22 Parameter ``iid`` is deprecated in 0.22 and will be removed in 0.24 - cv : int, cross-validation generator or an iterable, optional + cv : int, cross-validation generator or an iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: - - None, to use the default 5-fold cross validation, - integer, to specify the number of folds in a `(Stratified)KFold`, - :term:`CV splitter`, @@ -1265,20 +1282,20 @@ class RandomizedSearchCV(BaseSearchCV): .. versionchanged:: 0.22 ``cv`` default value if None changed from 3-fold to 5-fold. - refit : boolean, string, or callable, default=True + refit : bool, str, or callable, default=True Refit an estimator using the best found parameters on the whole dataset. - For multiple metric evaluation, this needs to be a string denoting the + For multiple metric evaluation, this needs to be a `str` denoting the scorer that would be used to find the best parameters for refitting the estimator at the end. Where there are considerations other than maximum score in choosing a best estimator, ``refit`` can be set to a function which returns the selected ``best_index_`` given the ``cv_results``. In that - case, the ``best_estimator_`` and ``best_parameters_`` will be set + case, the ``best_estimator_`` and ``best_params_`` will be set according to the returned ``best_index_`` while the ``best_score_`` - attribute will not be availble. + attribute will not be available. The refitted estimator is made available at the ``best_estimator_`` attribute and permits using ``predict`` directly on this @@ -1298,21 +1315,20 @@ class RandomizedSearchCV(BaseSearchCV): verbose : integer Controls the verbosity: the higher, the more messages. - random_state : int, RandomState instance or None, optional, default=None + random_state : int or RandomState instance, default=None Pseudo random number generator state used for random uniform sampling from lists of possible values instead of scipy.stats distributions. - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + Pass an int for reproducible output across multiple + function calls. + See :term:`Glossary `. - error_score : 'raise' or numeric + error_score : 'raise' or numeric, default=np.nan Value to assign to the score if an error occurs in estimator fitting. If set to 'raise', the error is raised. If a numeric value is given, FitFailedWarning is raised. This parameter does not affect the refit - step, which will always raise the error. Default is ``np.nan``. + step, which will always raise the error. - return_train_score : boolean, default=False + return_train_score : bool, default=False If ``False``, the ``cv_results_`` attribute will not include training scores. Computing training scores is used to get insights on how different diff --git a/sklearn/model_selection/_split.py b/sklearn/model_selection/_split.py index ff3a3ba5bf365..0b769aefe120c 100644 --- a/sklearn/model_selection/_split.py +++ b/sklearn/model_selection/_split.py @@ -56,14 +56,14 @@ def split(self, X, y=None, groups=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data, where n_samples is the number of samples and n_features is the number of features. - y : array-like, of length n_samples + y : array-like of shape (n_samples,) The target variable for supervised learning problems. - groups : array-like, with shape (n_samples,), optional + groups : array-like of shape (n_samples,), default=None Group labels for the samples used while splitting the dataset into train/test set. @@ -135,10 +135,10 @@ class LeaveOneOut(BaseCrossValidator): >>> print(loo) LeaveOneOut() >>> for train_index, test_index in loo.split(X): - ... print("TRAIN:", train_index, "TEST:", test_index) - ... X_train, X_test = X[train_index], X[test_index] - ... y_train, y_test = y[train_index], y[test_index] - ... print(X_train, X_test, y_train, y_test) + ... print("TRAIN:", train_index, "TEST:", test_index) + ... X_train, X_test = X[train_index], X[test_index] + ... y_train, y_test = y[train_index], y[test_index] + ... print(X_train, X_test, y_train, y_test) TRAIN: [1] TEST: [0] [[3 4]] [[1 2]] [2] [1] TRAIN: [0] TEST: [1] @@ -167,7 +167,7 @@ def get_n_splits(self, X, y=None, groups=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data, where n_samples is the number of samples and n_features is the number of features. @@ -222,9 +222,9 @@ class LeavePOut(BaseCrossValidator): >>> print(lpo) LeavePOut(p=2) >>> for train_index, test_index in lpo.split(X): - ... print("TRAIN:", train_index, "TEST:", test_index) - ... X_train, X_test = X[train_index], X[test_index] - ... y_train, y_test = y[train_index], y[test_index] + ... print("TRAIN:", train_index, "TEST:", test_index) + ... X_train, X_test = X[train_index], X[test_index] + ... y_train, y_test = y[train_index], y[test_index] TRAIN: [2 3] TEST: [0 1] TRAIN: [1 3] TEST: [0 2] TRAIN: [1 2] TEST: [0 3] @@ -251,7 +251,7 @@ def get_n_splits(self, X, y=None, groups=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data, where n_samples is the number of samples and n_features is the number of features. @@ -305,14 +305,14 @@ def split(self, X, y=None, groups=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data, where n_samples is the number of samples and n_features is the number of features. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,), default=None The target variable for supervised learning problems. - groups : array-like, with shape (n_samples,), optional + groups : array-like of shape (n_samples,), default=None Group labels for the samples used while splitting the dataset into train/test set. @@ -376,15 +376,16 @@ class KFold(_BaseKFold): .. versionchanged:: 0.22 ``n_splits`` default value changed from 3 to 5. - shuffle : boolean, optional + shuffle : bool, default=False Whether to shuffle the data before splitting into batches. + Note that the samples within each split will not be shuffled. - random_state : int, RandomState instance or None, optional, default=None - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. Only used when ``shuffle`` is True. This should be left - to None if ``shuffle`` is False. + random_state : int or RandomState instance, default=None + When `shuffle` is True, `random_state` affects the ordering of the + indices, which controls the randomness of each fold. Otherwise, this + parameter has no effect. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Examples -------- @@ -398,9 +399,9 @@ class KFold(_BaseKFold): >>> print(kf) KFold(n_splits=2, random_state=None, shuffle=False) >>> for train_index, test_index in kf.split(X): - ... print("TRAIN:", train_index, "TEST:", test_index) - ... X_train, X_test = X[train_index], X[test_index] - ... y_train, y_test = y[train_index], y[test_index] + ... print("TRAIN:", train_index, "TEST:", test_index) + ... X_train, X_test = X[train_index], X[test_index] + ... y_train, y_test = y[train_index], y[test_index] TRAIN: [2 3] TEST: [0 1] TRAIN: [0 1] TEST: [2 3] @@ -411,7 +412,7 @@ class KFold(_BaseKFold): ``n_samples // n_splits``, where ``n_samples`` is the number of samples. Randomized CV splitters may return different results for each call of - split. You can make the results identical by setting ``random_state`` + split. You can make the results identical by setting `random_state` to an integer. See also @@ -541,14 +542,14 @@ def split(self, X, y=None, groups=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data, where n_samples is the number of samples and n_features is the number of features. - y : array-like, shape (n_samples,), optional + y : array-like of shape (n_samples,), default=None The target variable for supervised learning problems. - groups : array-like, with shape (n_samples,) + groups : array-like of shape (n_samples,) Group labels for the samples used while splitting the dataset into train/test set. @@ -582,15 +583,16 @@ class StratifiedKFold(_BaseKFold): .. versionchanged:: 0.22 ``n_splits`` default value changed from 3 to 5. - shuffle : boolean, optional + shuffle : bool, default=False Whether to shuffle each class's samples before splitting into batches. + Note that the samples within each split will not be shuffled. - random_state : int, RandomState instance or None, optional, default=None - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. Only used when ``shuffle`` is True. This should be left - to None if ``shuffle`` is False. + random_state : int or RandomState instance, default=None + When `shuffle` is True, `random_state` affects the ordering of the + indices, which controls the randomness of each fold for each class. + Otherwise, leave `random_state` as `None`. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Examples -------- @@ -604,9 +606,9 @@ class StratifiedKFold(_BaseKFold): >>> print(skf) StratifiedKFold(n_splits=2, random_state=None, shuffle=False) >>> for train_index, test_index in skf.split(X, y): - ... print("TRAIN:", train_index, "TEST:", test_index) - ... X_train, X_test = X[train_index], X[test_index] - ... y_train, y_test = y[train_index], y[test_index] + ... print("TRAIN:", train_index, "TEST:", test_index) + ... X_train, X_test = X[train_index], X[test_index] + ... y_train, y_test = y[train_index], y[test_index] TRAIN: [1 3] TEST: [0 2] TRAIN: [0 2] TEST: [1 3] @@ -698,7 +700,7 @@ def split(self, X, y, groups=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data, where n_samples is the number of samples and n_features is the number of features. @@ -706,7 +708,7 @@ def split(self, X, y, groups=None): hence ``np.zeros(n_samples)`` may be used as a placeholder for ``X`` instead of actual training data. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) The target variable for supervised learning problems. Stratification is done based on the y labels. @@ -724,7 +726,7 @@ def split(self, X, y, groups=None): Notes ----- Randomized CV splitters may return different results for each call of - split. You can make the results identical by setting ``random_state`` + split. You can make the results identical by setting `random_state` to an integer. """ y = check_array(y, ensure_2d=False, dtype=None) @@ -756,7 +758,7 @@ class TimeSeriesSplit(_BaseKFold): .. versionchanged:: 0.22 ``n_splits`` default value changed from 3 to 5. - max_train_size : int, optional + max_train_size : int, default=None Maximum size for a single training set. Examples @@ -769,9 +771,9 @@ class TimeSeriesSplit(_BaseKFold): >>> print(tscv) TimeSeriesSplit(max_train_size=None, n_splits=5) >>> for train_index, test_index in tscv.split(X): - ... print("TRAIN:", train_index, "TEST:", test_index) - ... X_train, X_test = X[train_index], X[test_index] - ... y_train, y_test = y[train_index], y[test_index] + ... print("TRAIN:", train_index, "TEST:", test_index) + ... X_train, X_test = X[train_index], X[test_index] + ... y_train, y_test = y[train_index], y[test_index] TRAIN: [0] TEST: [1] TRAIN: [0 1] TEST: [2] TRAIN: [0 1 2] TEST: [3] @@ -794,14 +796,14 @@ def split(self, X, y=None, groups=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data, where n_samples is the number of samples and n_features is the number of features. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) Always ignored, exists for compatibility. - groups : array-like, with shape (n_samples,) + groups : array-like of shape (n_samples,) Always ignored, exists for compatibility. Yields @@ -861,10 +863,10 @@ class LeaveOneGroupOut(BaseCrossValidator): >>> print(logo) LeaveOneGroupOut() >>> for train_index, test_index in logo.split(X, y, groups): - ... print("TRAIN:", train_index, "TEST:", test_index) - ... X_train, X_test = X[train_index], X[test_index] - ... y_train, y_test = y[train_index], y[test_index] - ... print(X_train, X_test, y_train, y_test) + ... print("TRAIN:", train_index, "TEST:", test_index) + ... X_train, X_test = X[train_index], X[test_index] + ... y_train, y_test = y[train_index], y[test_index] + ... print(X_train, X_test, y_train, y_test) TRAIN: [2 3] TEST: [0 1] [[5 6] [7 8]] [[1 2] @@ -900,7 +902,7 @@ def get_n_splits(self, X=None, y=None, groups=None): y : object Always ignored, exists for compatibility. - groups : array-like, with shape (n_samples,) + groups : array-like of shape (n_samples,) Group labels for the samples used while splitting the dataset into train/test set. This 'groups' parameter must always be specified to calculate the number of splits, though the other parameters can be @@ -921,14 +923,14 @@ def split(self, X, y=None, groups=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data, where n_samples is the number of samples and n_features is the number of features. - y : array-like, of length n_samples, optional + y : array-like of shape (n_samples,), default=None The target variable for supervised learning problems. - groups : array-like, with shape (n_samples,) + groups : array-like of shape (n_samples,) Group labels for the samples used while splitting the dataset into train/test set. @@ -980,10 +982,10 @@ class LeavePGroupsOut(BaseCrossValidator): >>> print(lpgo) LeavePGroupsOut(n_groups=2) >>> for train_index, test_index in lpgo.split(X, y, groups): - ... print("TRAIN:", train_index, "TEST:", test_index) - ... X_train, X_test = X[train_index], X[test_index] - ... y_train, y_test = y[train_index], y[test_index] - ... print(X_train, X_test, y_train, y_test) + ... print("TRAIN:", train_index, "TEST:", test_index) + ... X_train, X_test = X[train_index], X[test_index] + ... y_train, y_test = y[train_index], y[test_index] + ... print(X_train, X_test, y_train, y_test) TRAIN: [2] TEST: [0 1] [[5 6]] [[1 2] [3 4]] [1] [1 2] @@ -1031,7 +1033,7 @@ def get_n_splits(self, X=None, y=None, groups=None): y : object Always ignored, exists for compatibility. - groups : array-like, with shape (n_samples,) + groups : array-like of shape (n_samples,) Group labels for the samples used while splitting the dataset into train/test set. This 'groups' parameter must always be specified to calculate the number of splits, though the other parameters can be @@ -1052,14 +1054,14 @@ def split(self, X, y=None, groups=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data, where n_samples is the number of samples and n_features is the number of features. - y : array-like, of length n_samples, optional + y : array-like of shape (n_samples,), default=None The target variable for supervised learning problems. - groups : array-like, with shape (n_samples,) + groups : array-like of shape (n_samples,) Group labels for the samples used while splitting the dataset into train/test set. @@ -1088,11 +1090,10 @@ class _RepeatedSplits(metaclass=ABCMeta): n_repeats : int, default=10 Number of times cross-validator needs to be repeated. - random_state : int, RandomState instance or None, optional, default=None - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState instance, default=None + Passes `random_state` to the arbitrary repeating cross validator. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. **cvargs : additional params Constructor parameters for cv. Must not contain random_state @@ -1123,10 +1124,10 @@ def split(self, X, y=None, groups=None): Training data, where n_samples is the number of samples and n_features is the number of features. - y : array-like, of length n_samples + y : array-like of length n_samples The target variable for supervised learning problems. - groups : array-like, with shape (n_samples,), optional + groups : array-like of shape (n_samples,), default=None Group labels for the samples used while splitting the dataset into train/test set. @@ -1160,7 +1161,7 @@ def get_n_splits(self, X=None, y=None, groups=None): Always ignored, exists for compatibility. ``np.zeros(n_samples)`` may be used as a placeholder. - groups : array-like, with shape (n_samples,), optional + groups : array-like of shape (n_samples,), default=None Group labels for the samples used while splitting the dataset into train/test set. @@ -1193,11 +1194,10 @@ class RepeatedKFold(_RepeatedSplits): n_repeats : int, default=10 Number of times cross-validator needs to be repeated. - random_state : int, RandomState instance or None, optional, default=None - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState instance, default=None + Controls the randomness of each repeated cross-validation instance. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Examples -------- @@ -1219,7 +1219,7 @@ class RepeatedKFold(_RepeatedSplits): Notes ----- Randomized CV splitters may return different results for each call of - split. You can make the results identical by setting ``random_state`` + split. You can make the results identical by setting `random_state` to an integer. See also @@ -1247,9 +1247,10 @@ class RepeatedStratifiedKFold(_RepeatedSplits): n_repeats : int, default=10 Number of times cross-validator needs to be repeated. - random_state : None, int or RandomState, default=None - Random state to be used to generate random state for each - repetition. + random_state : int or RandomState instance, default=None + Controls the generation of the random states for each repetition. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Examples -------- @@ -1272,7 +1273,7 @@ class RepeatedStratifiedKFold(_RepeatedSplits): Notes ----- Randomized CV splitters may return different results for each call of - split. You can make the results identical by setting ``random_state`` + split. You can make the results identical by setting `random_state` to an integer. See also @@ -1300,14 +1301,14 @@ def split(self, X, y=None, groups=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data, where n_samples is the number of samples and n_features is the number of features. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) The target variable for supervised learning problems. - groups : array-like, with shape (n_samples,), optional + groups : array-like of shape (n_samples,), default=None Group labels for the samples used while splitting the dataset into train/test set. @@ -1322,7 +1323,7 @@ def split(self, X, y=None, groups=None): Notes ----- Randomized CV splitters may return different results for each call of - split. You can make the results identical by setting ``random_state`` + split. You can make the results identical by setting `random_state` to an integer. """ X, y, groups = indexable(X, y, groups) @@ -1371,27 +1372,26 @@ class ShuffleSplit(BaseShuffleSplit): Parameters ---------- - n_splits : int, default 10 + n_splits : int, default=10 Number of re-shuffling & splitting iterations. - test_size : float, int, None, default=None + test_size : float or int, default=None If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to include in the test split. If int, represents the absolute number of test samples. If None, the value is set to the complement of the train size. If ``train_size`` is also None, it will be set to 0.1. - train_size : float, int, or None, default=None + train_size : float or int, default=None If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to include in the train split. If int, represents the absolute number of train samples. If None, the value is automatically set to the complement of the test size. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState instance, default=None + Controls the randomness of the training and testing indices produced. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Examples -------- @@ -1405,7 +1405,7 @@ class ShuffleSplit(BaseShuffleSplit): >>> print(rs) ShuffleSplit(n_splits=5, random_state=0, test_size=0.25, train_size=None) >>> for train_index, test_index in rs.split(X): - ... print("TRAIN:", train_index, "TEST:", test_index) + ... print("TRAIN:", train_index, "TEST:", test_index) TRAIN: [1 3 0 4] TEST: [5 2] TRAIN: [4 0 2 5] TEST: [1 3] TRAIN: [1 2 4 0] TEST: [3 5] @@ -1414,7 +1414,7 @@ class ShuffleSplit(BaseShuffleSplit): >>> rs = ShuffleSplit(n_splits=5, train_size=0.5, test_size=.25, ... random_state=0) >>> for train_index, test_index in rs.split(X): - ... print("TRAIN:", train_index, "TEST:", test_index) + ... print("TRAIN:", train_index, "TEST:", test_index) TRAIN: [1 3 0] TEST: [5 2] TRAIN: [4 0 2] TEST: [1 3] TRAIN: [1 2 4] TEST: [3 5] @@ -1470,30 +1470,28 @@ class GroupShuffleSplit(ShuffleSplit): Parameters ---------- - n_splits : int (default 5) + n_splits : int, default=5 Number of re-shuffling & splitting iterations. - test_size : float, int, None, optional (default=None) + test_size : float, int, default=0.2 If float, should be between 0.0 and 1.0 and represent the proportion of groups to include in the test split (rounded up). If int, represents the absolute number of test groups. If None, the value is - set to the complement of the train size. By default, the value is set - to 0.2. + set to the complement of the train size. The default will change in version 0.21. It will remain 0.2 only if ``train_size`` is unspecified, otherwise it will complement the specified ``train_size``. - train_size : float, int, or None, default is None + train_size : float or int, default=None If float, should be between 0.0 and 1.0 and represent the proportion of the groups to include in the train split. If int, represents the absolute number of train groups. If None, the value is automatically set to the complement of the test size. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState instance, default=None + Controls the randomness of the training and testing indices produced. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Examples -------- @@ -1508,7 +1506,7 @@ class GroupShuffleSplit(ShuffleSplit): >>> gss.get_n_splits() 2 >>> for train_idx, test_idx in gss.split(X, y, groups): - ... print("TRAIN:", train_idx, "TEST:", test_idx) + ... print("TRAIN:", train_idx, "TEST:", test_idx) TRAIN: [2 3 4 5 6 7] TEST: [0 1] TRAIN: [0 1 5 6 7] TEST: [2 3 4] ''' @@ -1541,14 +1539,14 @@ def split(self, X, y=None, groups=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data, where n_samples is the number of samples and n_features is the number of features. - y : array-like, shape (n_samples,), optional + y : array-like of shape (n_samples,), default=None The target variable for supervised learning problems. - groups : array-like, with shape (n_samples,) + groups : array-like of shape (n_samples,) Group labels for the samples used while splitting the dataset into train/test set. @@ -1563,7 +1561,7 @@ def split(self, X, y=None, groups=None): Notes ----- Randomized CV splitters may return different results for each call of - split. You can make the results identical by setting ``random_state`` + split. You can make the results identical by setting `random_state` to an integer. """ return super().split(X, y, groups) @@ -1586,27 +1584,26 @@ class StratifiedShuffleSplit(BaseShuffleSplit): Parameters ---------- - n_splits : int, default 10 + n_splits : int, default=10 Number of re-shuffling & splitting iterations. - test_size : float, int, None, optional (default=None) + test_size : float or int, default=None If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to include in the test split. If int, represents the absolute number of test samples. If None, the value is set to the complement of the train size. If ``train_size`` is also None, it will be set to 0.1. - train_size : float, int, or None, default is None + train_size : float or int, default=None If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to include in the train split. If int, represents the absolute number of train samples. If None, the value is automatically set to the complement of the test size. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState instance, default=None + Controls the randomness of the training and testing indices produced. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Examples -------- @@ -1620,9 +1617,9 @@ class StratifiedShuffleSplit(BaseShuffleSplit): >>> print(sss) StratifiedShuffleSplit(n_splits=5, random_state=0, ...) >>> for train_index, test_index in sss.split(X, y): - ... print("TRAIN:", train_index, "TEST:", test_index) - ... X_train, X_test = X[train_index], X[test_index] - ... y_train, y_test = y[train_index], y[test_index] + ... print("TRAIN:", train_index, "TEST:", test_index) + ... X_train, X_test = X[train_index], X[test_index] + ... y_train, y_test = y[train_index], y[test_index] TRAIN: [5 2 3] TEST: [4 1 0] TRAIN: [5 1 4] TEST: [0 2 3] TRAIN: [5 0 2] TEST: [4 3 1] @@ -1705,7 +1702,7 @@ def split(self, X, y, groups=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data, where n_samples is the number of samples and n_features is the number of features. @@ -1713,7 +1710,7 @@ def split(self, X, y, groups=None): hence ``np.zeros(n_samples)`` may be used as a placeholder for ``X`` instead of actual training data. - y : array-like, shape (n_samples,) or (n_samples, n_labels) + y : array-like of shape (n_samples,) or (n_samples, n_labels) The target variable for supervised learning problems. Stratification is done based on the y labels. @@ -1731,7 +1728,7 @@ def split(self, X, y, groups=None): Notes ----- Randomized CV splitters may return different results for each call of - split. You can make the results identical by setting ``random_state`` + split. You can make the results identical by setting `random_state` to an integer. """ y = check_array(y, ensure_2d=False, dtype=None) @@ -1816,9 +1813,11 @@ class PredefinedSplit(BaseCrossValidator): Read more in the :ref:`User Guide `. + .. versionadded:: 0.16 + Parameters ---------- - test_fold : array-like, shape (n_samples,) + test_fold : array-like of shape (n_samples,) The entry ``test_fold[i]`` represents the index of the test set that sample ``i`` belongs to. It is possible to exclude sample ``i`` from any test set (i.e. include sample ``i`` in every training set) by @@ -1837,9 +1836,9 @@ class PredefinedSplit(BaseCrossValidator): >>> print(ps) PredefinedSplit(test_fold=array([ 0, 1, -1, 1])) >>> for train_index, test_index in ps.split(): - ... print("TRAIN:", train_index, "TEST:", test_index) - ... X_train, X_test = X[train_index], X[test_index] - ... y_train, y_test = y[train_index], y[test_index] + ... print("TRAIN:", train_index, "TEST:", test_index) + ... X_train, X_test = X[train_index], X[test_index] + ... y_train, y_test = y[train_index], y[test_index] TRAIN: [1 2 3] TEST: [0] TRAIN: [0 2] TEST: [1 3] """ @@ -1965,11 +1964,10 @@ def check_cv(cv=5, y=None, classifier=False): Parameters ---------- - cv : int, cross-validation generator or an iterable, optional + cv : int, cross-validation generator or an iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: - - - None, to use the default 5-fold cross-validation, + - None, to use the default 5-fold cross validation, - integer, to specify the number of folds. - :term:`CV splitter`, - An iterable yielding (train, test) splits as arrays of indices. @@ -1984,10 +1982,10 @@ def check_cv(cv=5, y=None, classifier=False): .. versionchanged:: 0.22 ``cv`` default value changed from 3-fold to 5-fold. - y : array-like, optional + y : array-like, default=None The target variable for supervised learning problems. - classifier : boolean, optional, default False + classifier : bool, default=False Whether the task is a classification task, in which case stratified KFold will be used. @@ -2031,30 +2029,30 @@ def train_test_split(*arrays, **options): Allowed inputs are lists, numpy arrays, scipy-sparse matrices or pandas dataframes. - test_size : float, int or None, optional (default=None) + test_size : float or int, default=None If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to include in the test split. If int, represents the absolute number of test samples. If None, the value is set to the complement of the train size. If ``train_size`` is also None, it will be set to 0.25. - train_size : float, int, or None, (default=None) + train_size : float or int, default=None If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to include in the train split. If int, represents the absolute number of train samples. If None, the value is automatically set to the complement of the test size. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int or RandomState instance, default=None + Controls the shuffling applied to the data before applying the split. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. + - shuffle : boolean, optional (default=True) + shuffle : bool, default=True Whether or not to shuffle the data before splitting. If shuffle=False then stratify must be None. - stratify : array-like or None (default=None) + stratify : array-like, default=None If not None, data is split in a stratified fashion, using this as the class labels. diff --git a/sklearn/model_selection/_validation.py b/sklearn/model_selection/_validation.py index 88eb5d49c4d0f..ae6151a88727b 100644 --- a/sklearn/model_selection/_validation.py +++ b/sklearn/model_selection/_validation.py @@ -13,7 +13,7 @@ import warnings import numbers import time -from traceback import format_exception_only +from traceback import format_exc from contextlib import suppress import numpy as np @@ -23,7 +23,8 @@ from ..base import is_classifier, clone from ..utils import (indexable, check_random_state, _safe_indexing, _message_with_time) -from ..utils.validation import _is_arraylike, _num_samples +from ..utils.validation import _check_fit_params +from ..utils.validation import _num_samples from ..utils.metaestimators import _safe_split from ..metrics import check_scoring from ..metrics._scorer import _check_multimetric_scoring, _MultimetricScorer @@ -49,20 +50,21 @@ def cross_validate(estimator, X, y=None, groups=None, scoring=None, cv=None, estimator : estimator object implementing 'fit' The object to use to fit the data. - X : array-like + X : array-like of shape (n_samples, n_features) The data to fit. Can be for example a list, or an array. - y : array-like, optional, default: None + y : array-like of shape (n_samples,) or (n_samples, n_outputs), \ + default=None The target variable to try to predict in the case of supervised learning. - groups : array-like, with shape (n_samples,), optional + groups : array-like of shape (n_samples,), default=None Group labels for the samples used while splitting the dataset into train/test set. Only used in conjunction with a "Group" :term:`cv` instance (e.g., :class:`GroupKFold`). - scoring : string, callable, list/tuple, dict or None, default: None - A single string (see :ref:`scoring_parameter`) or a callable + scoring : str, callable, list/tuple, or dict, default=None + A single str (see :ref:`scoring_parameter`) or a callable (see :ref:`scoring`) to evaluate the predictions on the test set. For evaluating multiple metrics, either give a list of (unique) strings @@ -76,16 +78,16 @@ def cross_validate(estimator, X, y=None, groups=None, scoring=None, cv=None, If None, the estimator's score method is used. - cv : int, cross-validation generator or an iterable, optional + cv : int, cross-validation generator or an iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - - integer, to specify the number of folds in a `(Stratified)KFold`, + - int, to specify the number of folds in a `(Stratified)KFold`, - :term:`CV splitter`, - An iterable yielding (train, test) splits as arrays of indices. - For integer/None inputs, if the estimator is a classifier and ``y`` is + For int/None inputs, if the estimator is a classifier and ``y`` is either binary or multiclass, :class:`StratifiedKFold` is used. In all other cases, :class:`KFold` is used. @@ -95,19 +97,19 @@ def cross_validate(estimator, X, y=None, groups=None, scoring=None, cv=None, .. versionchanged:: 0.22 ``cv`` default value if None changed from 3-fold to 5-fold. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of CPUs to use to do the computation. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - verbose : integer, optional + verbose : int, default=0 The verbosity level. - fit_params : dict, optional + fit_params : dict, default=None Parameters to pass to the fit method of the estimator. - pre_dispatch : int, or string, optional + pre_dispatch : int or str, default='2*n_jobs' Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched @@ -121,10 +123,10 @@ def cross_validate(estimator, X, y=None, groups=None, scoring=None, cv=None, - An int, giving the exact number of total jobs that are spawned - - A string, giving an expression as a function of n_jobs, + - A str, giving an expression as a function of n_jobs, as in '2*n_jobs' - return_train_score : boolean, default=False + return_train_score : bool, default=False Whether to include train scores. Computing training scores is used to get insights on how different parameter settings impact the overfitting/underfitting trade-off. @@ -132,7 +134,7 @@ def cross_validate(estimator, X, y=None, groups=None, scoring=None, cv=None, expensive and is not strictly required to select the parameters that yield the best generalization performance. - return_estimator : boolean, default False + return_estimator : bool, default=False Whether to return the estimators fitted on each split. error_score : 'raise' or numeric @@ -271,20 +273,21 @@ def cross_val_score(estimator, X, y=None, groups=None, scoring=None, cv=None, estimator : estimator object implementing 'fit' The object to use to fit the data. - X : array-like + X : array-like of shape (n_samples, n_features) The data to fit. Can be for example a list, or an array. - y : array-like, optional, default: None + y : array-like of shape (n_samples,) or (n_samples, n_outputs), \ + default=None The target variable to try to predict in the case of supervised learning. - groups : array-like, with shape (n_samples,), optional + groups : array-like of shape (n_samples,), default=None Group labels for the samples used while splitting the dataset into train/test set. Only used in conjunction with a "Group" :term:`cv` instance (e.g., :class:`GroupKFold`). - scoring : string, callable or None, optional, default: None - A string (see model evaluation documentation) or + scoring : str or callable, default=None + A str (see model evaluation documentation) or a scorer callable object / function with signature ``scorer(estimator, X, y)`` which should return only a single value. @@ -294,16 +297,16 @@ def cross_val_score(estimator, X, y=None, groups=None, scoring=None, cv=None, If None, the estimator's default scorer (if available) is used. - cv : int, cross-validation generator or an iterable, optional + cv : int, cross-validation generator or an iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - - integer, to specify the number of folds in a `(Stratified)KFold`, + - int, to specify the number of folds in a `(Stratified)KFold`, - :term:`CV splitter`, - An iterable yielding (train, test) splits as arrays of indices. - For integer/None inputs, if the estimator is a classifier and ``y`` is + For int/None inputs, if the estimator is a classifier and ``y`` is either binary or multiclass, :class:`StratifiedKFold` is used. In all other cases, :class:`KFold` is used. @@ -313,19 +316,19 @@ def cross_val_score(estimator, X, y=None, groups=None, scoring=None, cv=None, .. versionchanged:: 0.22 ``cv`` default value if None changed from 3-fold to 5-fold. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of CPUs to use to do the computation. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - verbose : integer, optional + verbose : int, default=0 The verbosity level. - fit_params : dict, optional + fit_params : dict, default=None Parameters to pass to the fit method of the estimator. - pre_dispatch : int, or string, optional + pre_dispatch : int or str, default='2*n_jobs' Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched @@ -339,10 +342,10 @@ def cross_val_score(estimator, X, y=None, groups=None, scoring=None, cv=None, - An int, giving the exact number of total jobs that are spawned - - A string, giving an expression as a function of n_jobs, + - A str, giving an expression as a function of n_jobs, as in '2*n_jobs' - error_score : 'raise' or numeric + error_score : 'raise' or numeric, default=np.nan Value to assign to the score if an error occurs in estimator fitting. If set to 'raise', the error is raised. If a numeric value is given, FitFailedWarning is raised. This parameter @@ -402,10 +405,10 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, estimator : estimator object implementing 'fit' The object to use to fit the data. - X : array-like of shape at least 2D + X : array-like of shape (n_samples, n_features) The data to fit. - y : array-like, optional, default: None + y : array-like of shape (n_samples,) or (n_samples, n_outputs) or None The target variable to try to predict in the case of supervised learning. @@ -419,16 +422,16 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, The callable object / fn should have signature ``scorer(estimator, X, y)``. - train : array-like, shape (n_train_samples,) + train : array-like of shape (n_train_samples,) Indices of training samples. - test : array-like, shape (n_test_samples,) + test : array-like of shape (n_test_samples,) Indices of test samples. - verbose : integer + verbose : int The verbosity level. - error_score : 'raise' or numeric + error_score : 'raise' or numeric, default=np.nan Value to assign to the score if an error occurs in estimator fitting. If set to 'raise', the error is raised. If a numeric value is given, FitFailedWarning is raised. This parameter @@ -440,28 +443,28 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, fit_params : dict or None Parameters that will be passed to ``estimator.fit``. - return_train_score : boolean, optional, default: False + return_train_score : bool, default=False Compute and return score on training set. - return_parameters : boolean, optional, default: False + return_parameters : bool, default=False Return parameters that has been used for the estimator. - return_n_test_samples : boolean, optional, default: False + return_n_test_samples : bool, default=False Whether to return the ``n_test_samples`` - return_times : boolean, optional, default: False + return_times : bool, default=False Whether to return the fit/score times. - return_estimator : boolean, optional, default: False + return_estimator : bool, default=False Whether to return the fitted estimator. Returns ------- - train_scores : dict of scorer name -> float, optional + train_scores : dict of scorer name -> float Score on training set (for all the scorers), returned only if `return_train_score` is `True`. - test_scores : dict of scorer name -> float, optional + test_scores : dict of scorer name -> float Score on testing set (for all the scorers). n_test_samples : int @@ -473,7 +476,7 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, score_time : float Time spent for scoring in seconds. - parameters : dict or None, optional + parameters : dict or None The parameters that have been evaluated. estimator : estimator object @@ -489,8 +492,7 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, # Adjust length of sample weights fit_params = fit_params if fit_params is not None else {} - fit_params = {k: _index_param_value(X, v, train) - for k, v in fit_params.items()} + fit_params = _check_fit_params(X, fit_params, train) train_scores = {} if parameters is not None: @@ -532,7 +534,7 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, warnings.warn("Estimator fit failed. The score on this train-test" " partition for these parameters will be set to %f. " "Details: \n%s" % - (error_score, format_exception_only(type(e), e)[0]), + (error_score, format_exc()), FitFailedWarning) else: raise ValueError("error_score must be the string 'raise' or a" @@ -632,28 +634,29 @@ def cross_val_predict(estimator, X, y=None, groups=None, cv=None, estimator : estimator object implementing 'fit' and 'predict' The object to use to fit the data. - X : array-like + X : array-like of shape (n_samples, n_features) The data to fit. Can be, for example a list, or an array at least 2d. - y : array-like, optional, default: None + y : array-like of shape (n_samples,) or (n_samples, n_outputs), \ + default=None The target variable to try to predict in the case of supervised learning. - groups : array-like, with shape (n_samples,), optional + groups : array-like of shape (n_samples,), default=None Group labels for the samples used while splitting the dataset into train/test set. Only used in conjunction with a "Group" :term:`cv` instance (e.g., :class:`GroupKFold`). - cv : int, cross-validation generator or an iterable, optional + cv : int, cross-validation generator or an iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - - integer, to specify the number of folds in a `(Stratified)KFold`, + - int, to specify the number of folds in a `(Stratified)KFold`, - :term:`CV splitter`, - An iterable yielding (train, test) splits as arrays of indices. - For integer/None inputs, if the estimator is a classifier and ``y`` is + For int/None inputs, if the estimator is a classifier and ``y`` is either binary or multiclass, :class:`StratifiedKFold` is used. In all other cases, :class:`KFold` is used. @@ -663,19 +666,19 @@ def cross_val_predict(estimator, X, y=None, groups=None, cv=None, .. versionchanged:: 0.22 ``cv`` default value if None changed from 3-fold to 5-fold. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of CPUs to use to do the computation. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - verbose : integer, optional + verbose : int, default=0 The verbosity level. - fit_params : dict, optional + fit_params : dict, defualt=None Parameters to pass to the fit method of the estimator. - pre_dispatch : int, or string, optional + pre_dispatch : int or str, default='2*n_jobs' Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched @@ -689,10 +692,10 @@ def cross_val_predict(estimator, X, y=None, groups=None, cv=None, - An int, giving the exact number of total jobs that are spawned - - A string, giving an expression as a function of n_jobs, + - A str, giving an expression as a function of n_jobs, as in '2*n_jobs' - method : string, optional, default: 'predict' + method : str, default='predict' Invokes the passed method name of the passed estimator. For method='predict_proba', the columns correspond to the classes in sorted order. @@ -734,7 +737,7 @@ def cross_val_predict(estimator, X, y=None, groups=None, cv=None, # If classification methods produce multiple columns of output, # we need to manually encode classes to ensure consistent column ordering. encode = method in ['decision_function', 'predict_proba', - 'predict_log_proba'] + 'predict_log_proba'] and y is not None if encode: y = np.asarray(y) if y.ndim == 1: @@ -798,26 +801,26 @@ def _fit_and_predict(estimator, X, y, train, test, verbose, fit_params, estimator : estimator object implementing 'fit' and 'predict' The object to use to fit the data. - X : array-like of shape at least 2D + X : array-like of shape (n_samples, n_features) The data to fit. - y : array-like, optional, default: None + y : array-like of shape (n_samples,) or (n_samples, n_outputs) or None The target variable to try to predict in the case of supervised learning. - train : array-like, shape (n_train_samples,) + train : array-like of shape (n_train_samples,) Indices of training samples. - test : array-like, shape (n_test_samples,) + test : array-like of shape (n_test_samples,) Indices of test samples. - verbose : integer + verbose : int The verbosity level. fit_params : dict or None Parameters that will be passed to ``estimator.fit``. - method : string + method : str Invokes the passed method name of the passed estimator. Returns @@ -830,8 +833,7 @@ def _fit_and_predict(estimator, X, y, train, test, verbose, fit_params, """ # Adjust length of sample weights fit_params = fit_params if fit_params is not None else {} - fit_params = {k: _index_param_value(X, v, train) - for k, v in fit_params.items()} + fit_params = _check_fit_params(X, fit_params, train) X_train, y_train = _safe_split(estimator, X, y, train) X_test, _ = _safe_split(estimator, X, y, test, train) @@ -842,7 +844,11 @@ def _fit_and_predict(estimator, X, y, train, test, verbose, fit_params, estimator.fit(X_train, y_train, **fit_params) func = getattr(estimator, method) predictions = func(X_test) - if method in ['decision_function', 'predict_proba', 'predict_log_proba']: + + encode = method in ['decision_function', 'predict_proba', + 'predict_log_proba'] and y is not None + + if encode: if isinstance(predictions, list): predictions = [_enforce_prediction_order( estimator.classes_[i_label], predictions[i_label], @@ -863,7 +869,7 @@ def _enforce_prediction_order(classes, predictions, n_classes, method): not present in the subset of data used for training, then the output prediction array might not have the same columns as other folds. Use the list of class names - (assumed to be integers) to enforce the correct column order. + (assumed to be ints) to enforce the correct column order. Note that `classes` is the list of classes in this fold (a subset of the classes in the full training set) @@ -919,7 +925,7 @@ def _check_is_permutation(indices, n_samples): Parameters ---------- indices : ndarray - integer array to test + int array to test n_samples : int number of expected elements @@ -937,16 +943,6 @@ def _check_is_permutation(indices, n_samples): return True -def _index_param_value(X, v, indices): - """Private helper function for parameter value indexing.""" - if not _is_arraylike(v) or _num_samples(v) != _num_samples(X): - # pass through: skip indexing - return v - if sp.issparse(v): - v = v.tocsr() - return _safe_indexing(v, indices) - - def permutation_test_score(estimator, X, y, groups=None, cv=None, n_permutations=100, n_jobs=None, random_state=0, verbose=0, scoring=None): @@ -962,11 +958,11 @@ def permutation_test_score(estimator, X, y, groups=None, cv=None, X : array-like of shape at least 2D The data to fit. - y : array-like + y : array-like of shape (n_samples,) or (n_samples, n_outputs) or None The target variable to try to predict in the case of supervised learning. - groups : array-like, with shape (n_samples,), optional + groups : array-like of shape (n_samples,), default=None Labels to constrain permutation within groups, i.e. ``y`` values are permuted among samples with the same group identifier. When not specified, ``y`` values are permuted among all samples. @@ -976,22 +972,22 @@ def permutation_test_score(estimator, X, y, groups=None, cv=None, cross-validator uses them for grouping the samples while splitting the dataset into train/test set. - scoring : string, callable or None, optional, default: None - A single string (see :ref:`scoring_parameter`) or a callable + scoring : str or callable, default=None + A single str (see :ref:`scoring_parameter`) or a callable (see :ref:`scoring`) to evaluate the predictions on the test set. If None the estimator's score method is used. - cv : int, cross-validation generator or an iterable, optional + cv : int, cross-validation generator or an iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - - integer, to specify the number of folds in a `(Stratified)KFold`, + - int, to specify the number of folds in a `(Stratified)KFold`, - :term:`CV splitter`, - An iterable yielding (train, test) splits as arrays of indices. - For integer/None inputs, if the estimator is a classifier and ``y`` is + For int/None inputs, if the estimator is a classifier and ``y`` is either binary or multiclass, :class:`StratifiedKFold` is used. In all other cases, :class:`KFold` is used. @@ -1001,22 +997,20 @@ def permutation_test_score(estimator, X, y, groups=None, cv=None, .. versionchanged:: 0.22 ``cv`` default value if None changed from 3-fold to 5-fold. - n_permutations : integer, optional + n_permutations : int, default=100 Number of times to permute ``y``. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of CPUs to use to do the computation. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - random_state : int, RandomState instance or None, optional (default=0) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance or None, default=0 + Pass an int for reproducible output for permutation of + ``y`` values among samples. See :term:`Glossary `. - verbose : integer, optional + verbose : int, default=0 The verbosity level. Returns @@ -1024,7 +1018,7 @@ def permutation_test_score(estimator, X, y, groups=None, cv=None, score : float The true score without permuting targets. - permutation_scores : array, shape (n_permutations,) + permutation_scores : array of shape (n_permutations,) The scores obtained for each permutations. pvalue : float @@ -1044,6 +1038,7 @@ def permutation_test_score(estimator, X, y, groups=None, cv=None, Ojala and Garriga. Permutation Tests for Studying Classifier Performance. The Journal of Machine Learning Research (2010) vol. 11 + `[pdf] `_. """ X, y, groups = indexable(X, y, groups) @@ -1111,20 +1106,21 @@ def learning_curve(estimator, X, y, groups=None, estimator : object type that implements the "fit" and "predict" methods An object of that type which is cloned for each validation. - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training vector, where n_samples is the number of samples and n_features is the number of features. - y : array-like, shape (n_samples) or (n_samples, n_features), optional + y : array-like of shape (n_samples,) or (n_samples, n_outputs) Target relative to X for classification or regression; None for unsupervised learning. - groups : array-like, with shape (n_samples,), optional + groups : array-like of shape (n_samples,), default=None Group labels for the samples used while splitting the dataset into train/test set. Only used in conjunction with a "Group" :term:`cv` instance (e.g., :class:`GroupKFold`). - train_sizes : array-like, shape (n_ticks,), dtype float or int + train_sizes : array-like of shape (n_ticks,), \ + default=np.linspace(0.1, 1.0, 5) Relative or absolute numbers of training examples that will be used to generate the learning curve. If the dtype is float, it is regarded as a fraction of the maximum size of the training set (that is determined @@ -1132,18 +1128,17 @@ def learning_curve(estimator, X, y, groups=None, Otherwise it is interpreted as absolute sizes of the training sets. Note that for classification the number of samples usually have to be big enough to contain at least one sample from each class. - (default: np.linspace(0.1, 1.0, 5)) - cv : int, cross-validation generator or an iterable, optional + cv : int, cross-validation generator or an iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - - integer, to specify the number of folds in a `(Stratified)KFold`, + - int, to specify the number of folds in a `(Stratified)KFold`, - :term:`CV splitter`, - An iterable yielding (train, test) splits as arrays of indices. - For integer/None inputs, if the estimator is a classifier and ``y`` is + For int/None inputs, if the estimator is a classifier and ``y`` is either binary or multiclass, :class:`StratifiedKFold` is used. In all other cases, :class:`KFold` is used. @@ -1153,66 +1148,65 @@ def learning_curve(estimator, X, y, groups=None, .. versionchanged:: 0.22 ``cv`` default value if None changed from 3-fold to 5-fold. - scoring : string, callable or None, optional, default: None - A string (see model evaluation documentation) or + scoring : str or callable, default=None + A str (see model evaluation documentation) or a scorer callable object / function with signature ``scorer(estimator, X, y)``. - exploit_incremental_learning : boolean, optional, default: False + exploit_incremental_learning : bool, default=False If the estimator supports incremental learning, this will be used to speed up fitting for different training set sizes. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None Number of jobs to run in parallel. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - pre_dispatch : integer or string, optional + pre_dispatch : int or str, default='all' Number of predispatched jobs for parallel execution (default is - all). The option can reduce the allocated memory. The string can + all). The option can reduce the allocated memory. The str can be an expression like '2*n_jobs'. - verbose : integer, optional + verbose : int, default=0 Controls the verbosity: the higher, the more messages. - shuffle : boolean, optional + shuffle : bool, default=False Whether to shuffle training data before taking prefixes of it based on``train_sizes``. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. Used when ``shuffle`` is True. + random_state : int or RandomState instance, default=None + Used when ``shuffle`` is True. Pass an int for reproducible + output across multiple function calls. + See :term:`Glossary `. - error_score : 'raise' or numeric + error_score : 'raise' or numeric, default=np.nan Value to assign to the score if an error occurs in estimator fitting. If set to 'raise', the error is raised. If a numeric value is given, FitFailedWarning is raised. This parameter does not affect the refit step, which will always raise the error. - return_times : boolean, optional (default: False) + return_times : bool, default=False Whether to return the fit and score times. Returns ------- - train_sizes_abs : array, shape (n_unique_ticks,), dtype int + train_sizes_abs : array of shape (n_unique_ticks,) Numbers of training examples that has been used to generate the learning curve. Note that the number of ticks might be less than n_ticks because duplicate entries will be removed. - train_scores : array, shape (n_ticks, n_cv_folds) + train_scores : array of shape (n_ticks, n_cv_folds) Scores on training sets. - test_scores : array, shape (n_ticks, n_cv_folds) + test_scores : array of shape (n_ticks, n_cv_folds) Scores on test set. - fit_times : array, shape (n_ticks, n_cv_folds) + fit_times : array of shape (n_ticks, n_cv_folds) Times spent for fitting in seconds. Only present if ``return_times`` is True. - score_times : array, shape (n_ticks, n_cv_folds) + score_times : array of shape (n_ticks, n_cv_folds) Times spent for scoring in seconds. Only present if ``return_times`` is True. @@ -1289,7 +1283,7 @@ def _translate_train_sizes(train_sizes, n_max_training_samples): Parameters ---------- - train_sizes : array-like, shape (n_ticks,), dtype float or int + train_sizes : array-like of shape (n_ticks,) Numbers of training examples that will be used to generate the learning curve. If the dtype is float, it is regarded as a fraction of 'n_max_training_samples', i.e. it has to be within (0, 1]. @@ -1299,7 +1293,7 @@ def _translate_train_sizes(train_sizes, n_max_training_samples): Returns ------- - train_sizes_abs : array, shape (n_unique_ticks,), dtype int + train_sizes_abs : array of shape (n_unique_ticks,) Numbers of training examples that will be used to generate the learning curve. Note that the number of ticks might be less than n_ticks because duplicate entries will be removed. @@ -1392,35 +1386,35 @@ def validation_curve(estimator, X, y, param_name, param_range, groups=None, estimator : object type that implements the "fit" and "predict" methods An object of that type which is cloned for each validation. - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training vector, where n_samples is the number of samples and n_features is the number of features. - y : array-like, shape (n_samples) or (n_samples, n_features), optional + y : array-like of shape (n_samples,) or (n_samples, n_outputs) or None Target relative to X for classification or regression; None for unsupervised learning. - param_name : string + param_name : str Name of the parameter that will be varied. - param_range : array-like, shape (n_values,) + param_range : array-like of shape (n_values,) The values of the parameter that will be evaluated. - groups : array-like, with shape (n_samples,), optional + groups : array-like of shape (n_samples,), default=None Group labels for the samples used while splitting the dataset into train/test set. Only used in conjunction with a "Group" :term:`cv` instance (e.g., :class:`GroupKFold`). - cv : int, cross-validation generator or an iterable, optional + cv : int, cross-validation generator or an iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - - integer, to specify the number of folds in a `(Stratified)KFold`, + - int, to specify the number of folds in a `(Stratified)KFold`, - :term:`CV splitter`, - An iterable yielding (train, test) splits as arrays of indices. - For integer/None inputs, if the estimator is a classifier and ``y`` is + For int/None inputs, if the estimator is a classifier and ``y`` is either binary or multiclass, :class:`StratifiedKFold` is used. In all other cases, :class:`KFold` is used. @@ -1430,26 +1424,26 @@ def validation_curve(estimator, X, y, param_name, param_range, groups=None, .. versionchanged:: 0.22 ``cv`` default value if None changed from 3-fold to 5-fold. - scoring : string, callable or None, optional, default: None - A string (see model evaluation documentation) or + scoring : str or callable, default=None + A str (see model evaluation documentation) or a scorer callable object / function with signature ``scorer(estimator, X, y)``. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None Number of jobs to run in parallel. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - pre_dispatch : integer or string, optional + pre_dispatch : int or str, default='all' Number of predispatched jobs for parallel execution (default is - all). The option can reduce the allocated memory. The string can + all). The option can reduce the allocated memory. The str can be an expression like '2*n_jobs'. - verbose : integer, optional + verbose : int, default=0 Controls the verbosity: the higher, the more messages. - error_score : 'raise' or numeric + error_score : 'raise' or numeric, default=np.nan Value to assign to the score if an error occurs in estimator fitting. If set to 'raise', the error is raised. If a numeric value is given, FitFailedWarning is raised. This parameter @@ -1457,10 +1451,10 @@ def validation_curve(estimator, X, y, param_name, param_range, groups=None, Returns ------- - train_scores : array, shape (n_ticks, n_cv_folds) + train_scores : array of shape (n_ticks, n_cv_folds) Scores on training sets. - test_scores : array, shape (n_ticks, n_cv_folds) + test_scores : array of shape (n_ticks, n_cv_folds) Scores on test set. Notes diff --git a/sklearn/model_selection/tests/test_search.py b/sklearn/model_selection/tests/test_search.py index fc6183f3a1f0b..49d4b156e0686 100644 --- a/sklearn/model_selection/tests/test_search.py +++ b/sklearn/model_selection/tests/test_search.py @@ -27,7 +27,7 @@ from scipy.stats import bernoulli, expon, uniform -from sklearn.base import BaseEstimator +from sklearn.base import BaseEstimator, ClassifierMixin from sklearn.base import clone from sklearn.exceptions import NotFittedError from sklearn.datasets import make_classification @@ -36,6 +36,7 @@ from sklearn.model_selection import fit_grid_point from sklearn.model_selection import cross_val_score +from sklearn.model_selection import train_test_split from sklearn.model_selection import KFold from sklearn.model_selection import StratifiedKFold from sklearn.model_selection import StratifiedShuffleSplit @@ -66,6 +67,8 @@ from sklearn.impute import SimpleImputer from sklearn.pipeline import Pipeline from sklearn.linear_model import Ridge, SGDClassifier, LinearRegression +from sklearn.experimental import enable_hist_gradient_boosting # noqa +from sklearn.ensemble import HistGradientBoostingClassifier from sklearn.model_selection.tests.common import OneTimeSplitter @@ -218,33 +221,25 @@ def test_grid_search_pipeline_steps(): assert not hasattr(param_grid['regressor'][1], 'coef_') -def check_hyperparameter_searcher_with_fit_params(klass, **klass_kwargs): +@pytest.mark.parametrize("SearchCV", [GridSearchCV, RandomizedSearchCV]) +def test_SearchCV_with_fit_params(SearchCV): X = np.arange(100).reshape(10, 10) y = np.array([0] * 5 + [1] * 5) clf = CheckingClassifier(expected_fit_params=['spam', 'eggs']) - searcher = klass(clf, {'foo_param': [1, 2, 3]}, cv=2, **klass_kwargs) + searcher = SearchCV( + clf, {'foo_param': [1, 2, 3]}, cv=2, error_score="raise" + ) # The CheckingClassifier generates an assertion error if # a parameter is missing or has length != len(X). - assert_raise_message(AssertionError, - "Expected fit parameter(s) ['eggs'] not seen.", - searcher.fit, X, y, spam=np.ones(10)) - assert_raise_message( - ValueError, - "Found input variables with inconsistent numbers of samples: [", - searcher.fit, X, y, spam=np.ones(1), - eggs=np.zeros(10)) - searcher.fit(X, y, spam=np.ones(10), eggs=np.zeros(10)) - - -def test_grid_search_with_fit_params(): - check_hyperparameter_searcher_with_fit_params(GridSearchCV, - error_score='raise') + err_msg = r"Expected fit parameter\(s\) \['eggs'\] not seen." + with pytest.raises(AssertionError, match=err_msg): + searcher.fit(X, y, spam=np.ones(10)) - -def test_random_search_with_fit_params(): - check_hyperparameter_searcher_with_fit_params(RandomizedSearchCV, n_iter=1, - error_score='raise') + err_msg = "Fit parameter spam has length 1; expected" + with pytest.raises(AssertionError, match=err_msg): + searcher.fit(X, y, spam=np.ones(1), eggs=np.zeros(10)) + searcher.fit(X, y, spam=np.ones(10), eggs=np.zeros(10)) @ignore_warnings @@ -1126,7 +1121,6 @@ def test_random_search_cv_results_multimetric(): n_splits = 3 n_search_iter = 30 - scoring = ('accuracy', 'recall') # Scipy 0.12's stats dists do not accept seed, hence we use param grid params = dict(C=np.logspace(-4, 1, 3), @@ -1152,9 +1146,8 @@ def test_random_search_cv_results_multimetric(): compare_cv_results_multimetric_with_single(*random_searches, iid=iid) - if refit: - compare_refit_methods_when_refit_with_acc( - random_searches[0], random_searches[1], refit) + compare_refit_methods_when_refit_with_acc( + random_searches[0], random_searches[1], refit) @pytest.mark.filterwarnings("ignore:The parameter 'iid' is deprecated") # 0.24 @@ -1191,11 +1184,12 @@ def compare_cv_results_multimetric_with_single( def compare_refit_methods_when_refit_with_acc(search_multi, search_acc, refit): """Compare refit multi-metric search methods with single metric methods""" + assert search_acc.refit == refit if refit: assert search_multi.refit == 'accuracy' else: assert not search_multi.refit - assert search_acc.refit == refit + return # search cannot predict/score without refit X, y = make_blobs(n_samples=100, n_features=4, random_state=42) for method in ('predict', 'predict_proba', 'predict_log_proba'): @@ -1313,6 +1307,8 @@ def test_grid_search_correct_score_results(): assert_almost_equal(correct_score, cv_scores[i]) +# FIXME remove test_fit_grid_point as the function will be removed on 0.25 +@ignore_warnings(category=FutureWarning) def test_fit_grid_point(): X, y = make_classification(random_state=0) cv = StratifiedKFold() @@ -1341,6 +1337,21 @@ def test_fit_grid_point(): {'score': scorer}, verbose=True) +# FIXME remove test_fit_grid_point_deprecated as +# fit_grid_point will be removed on 0.25 +def test_fit_grid_point_deprecated(): + X, y = make_classification(random_state=0) + svc = LinearSVC(random_state=0) + scorer = make_scorer(accuracy_score) + msg = ("fit_grid_point is deprecated in version 0.23 " + "and will be removed in version 0.25") + params = {'C': 0.1} + train, test = next(StratifiedKFold().split(X, y)) + + with pytest.warns(FutureWarning, match=msg): + fit_grid_point(X, y, svc, params, train, test, scorer, verbose=False) + + def test_pickle(): # Test that a fit search can be pickled clf = MockClassifier() @@ -1358,7 +1369,6 @@ def test_pickle(): random_search_pickled.predict(X)) -@pytest.mark.filterwarnings('ignore: The default value of multioutput') # 0.23 def test_grid_search_with_multioutput_data(): # Test search with multi-output estimator @@ -1802,6 +1812,23 @@ def get_n_splits(self, *args, **kw): ridge.fit(X[:train_size], y[:train_size]) +def test_n_features_in(): + # make sure grid search and random search delegate n_features_in to the + # best estimator + n_features = 4 + X, y = make_classification(n_features=n_features) + gbdt = HistGradientBoostingClassifier() + param_grid = {'max_iter': [3, 4]} + gs = GridSearchCV(gbdt, param_grid) + rs = RandomizedSearchCV(gbdt, param_grid, n_iter=1) + assert not hasattr(gs, 'n_features_in_') + assert not hasattr(rs, 'n_features_in_') + gs.fit(X, y) + rs.fit(X, y) + assert gs.n_features_in_ == n_features + assert rs.n_features_in_ == n_features + + def test_search_cv__pairwise_property_delegated_to_base_estimator(): """ Test implementation of BaseSearchCV has the _pairwise property @@ -1847,3 +1874,78 @@ def test_search_cv__pairwise_property_equivalence_of_precomputed(): attr_message = "GridSearchCV not identical with precomputed metric" assert (preds_original == preds_precomputed).all(), attr_message + + +@pytest.mark.parametrize( + "SearchCV, param_search", + [(GridSearchCV, {'a': [0.1, 0.01]}), + (RandomizedSearchCV, {'a': uniform(1, 3)})] +) +def test_scalar_fit_param(SearchCV, param_search): + # unofficially sanctioned tolerance for scalar values in fit_params + # non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/15805 + class TestEstimator(BaseEstimator, ClassifierMixin): + def __init__(self, a=None): + self.a = a + + def fit(self, X, y, r=None): + self.r_ = r + + def predict(self, X): + return np.zeros(shape=(len(X))) + + model = SearchCV(TestEstimator(), param_search) + X, y = make_classification(random_state=42) + model.fit(X, y, r=42) + assert model.best_estimator_.r_ == 42 + + +@pytest.mark.parametrize( + "SearchCV, param_search", + [(GridSearchCV, {'alpha': [0.1, 0.01]}), + (RandomizedSearchCV, {'alpha': uniform(0.01, 0.1)})] +) +def test_scalar_fit_param_compat(SearchCV, param_search): + # check support for scalar values in fit_params, for instance in LightGBM + # that do not exactly respect the scikit-learn API contract but that we do + # not want to break without an explicit deprecation cycle and API + # recommendations for implementing early stopping with a user provided + # validation set. non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/15805 + X_train, X_valid, y_train, y_valid = train_test_split( + *make_classification(random_state=42), random_state=42 + ) + + class _FitParamClassifier(SGDClassifier): + + def fit(self, X, y, sample_weight=None, tuple_of_arrays=None, + scalar_param=None, callable_param=None): + super().fit(X, y, sample_weight=sample_weight) + assert scalar_param > 0 + assert callable(callable_param) + + # The tuple of arrays should be preserved as tuple. + assert isinstance(tuple_of_arrays, tuple) + assert tuple_of_arrays[0].ndim == 2 + assert tuple_of_arrays[1].ndim == 1 + return self + + def _fit_param_callable(): + pass + + model = SearchCV( + _FitParamClassifier(), param_search + ) + + # NOTE: `fit_params` should be data dependent (e.g. `sample_weight`) which + # is not the case for the following parameters. But this abuse is common in + # popular third-party libraries and we should tolerate this behavior for + # now and be careful not to break support for those without following + # proper deprecation cycle. + fit_params = { + 'tuple_of_arrays': (X_valid, y_valid), + 'callable_param': _fit_param_callable, + 'scalar_param': 42, + } + model.fit(X_train, y_train, **fit_params) diff --git a/sklearn/model_selection/tests/test_split.py b/sklearn/model_selection/tests/test_split.py index 5d3c41900472e..875e113f8dc36 100644 --- a/sklearn/model_selection/tests/test_split.py +++ b/sklearn/model_selection/tests/test_split.py @@ -63,69 +63,6 @@ digits = load_digits() -class MockClassifier: - """Dummy classifier to test the cross-validation""" - - def __init__(self, a=0, allow_nd=False): - self.a = a - self.allow_nd = allow_nd - - def fit(self, X, Y=None, sample_weight=None, class_prior=None, - sparse_sample_weight=None, sparse_param=None, dummy_int=None, - dummy_str=None, dummy_obj=None, callback=None): - """The dummy arguments are to test that this fit function can - accept non-array arguments through cross-validation, such as: - - int - - str (this is actually array-like) - - object - - function - """ - self.dummy_int = dummy_int - self.dummy_str = dummy_str - self.dummy_obj = dummy_obj - if callback is not None: - callback(self) - - if self.allow_nd: - X = X.reshape(len(X), -1) - if X.ndim >= 3 and not self.allow_nd: - raise ValueError('X cannot be d') - if sample_weight is not None: - assert sample_weight.shape[0] == X.shape[0], ( - 'MockClassifier extra fit_param sample_weight.shape[0]' - ' is {0}, should be {1}'.format(sample_weight.shape[0], - X.shape[0])) - if class_prior is not None: - assert class_prior.shape[0] == len(np.unique(y)), ( - 'MockClassifier extra fit_param class_prior.shape[0]' - ' is {0}, should be {1}'.format(class_prior.shape[0], - len(np.unique(y)))) - if sparse_sample_weight is not None: - fmt = ('MockClassifier extra fit_param sparse_sample_weight' - '.shape[0] is {0}, should be {1}') - assert sparse_sample_weight.shape[0] == X.shape[0], \ - fmt.format(sparse_sample_weight.shape[0], X.shape[0]) - if sparse_param is not None: - fmt = ('MockClassifier extra fit_param sparse_param.shape ' - 'is ({0}, {1}), should be ({2}, {3})') - assert sparse_param.shape == P_sparse.shape, ( - fmt.format(sparse_param.shape[0], - sparse_param.shape[1], - P_sparse.shape[0], P_sparse.shape[1])) - return self - - def predict(self, T): - if self.allow_nd: - T = T.reshape(len(T), -1) - return T[:, 0] - - def score(self, X=None, Y=None): - return 1. / (1 + np.abs(self.a)) - - def get_params(self, deep=False): - return {'a': self.a, 'allow_nd': self.allow_nd} - - @ignore_warnings def test_cross_validator_with_default_params(): n_samples = 4 @@ -175,7 +112,7 @@ def test_cross_validator_with_default_params(): # Test that train, test indices returned are integers for train, test in cv.split(X, y, groups): assert np.asarray(train).dtype.kind == 'i' - assert np.asarray(train).dtype.kind == 'i' + assert np.asarray(test).dtype.kind == 'i' # Test if the repr works without any errors assert cv_repr == repr(cv) @@ -227,13 +164,10 @@ def check_valid_split(train, test, n_samples=None): assert train.union(test) == set(range(n_samples)) -def check_cv_coverage(cv, X, y, groups, expected_n_splits=None): +def check_cv_coverage(cv, X, y, groups, expected_n_splits): n_samples = _num_samples(X) # Check that a all the samples appear at least once in a test fold - if expected_n_splits is not None: - assert cv.get_n_splits(X, y, groups) == expected_n_splits - else: - expected_n_splits = cv.get_n_splits(X, y, groups) + assert cv.get_n_splits(X, y, groups) == expected_n_splits collected_test_samples = set() iterations = 0 @@ -1338,16 +1272,16 @@ def test_cv_iterable_wrapper(): list(kf_iter_wrapped.split(X, y))) # If the splits are randomized, successive calls to split yields different # results - kf_randomized_iter = KFold(shuffle=True).split(X, y) + kf_randomized_iter = KFold(shuffle=True, random_state=0).split(X, y) kf_randomized_iter_wrapped = check_cv(kf_randomized_iter) # numpy's assert_array_equal properly compares nested lists np.testing.assert_equal(list(kf_randomized_iter_wrapped.split(X, y)), list(kf_randomized_iter_wrapped.split(X, y))) try: + splits_are_equal = True np.testing.assert_equal(list(kf_iter_wrapped.split(X, y)), list(kf_randomized_iter_wrapped.split(X, y))) - splits_are_equal = True except AssertionError: splits_are_equal = False assert not splits_are_equal, ( diff --git a/sklearn/model_selection/tests/test_validation.py b/sklearn/model_selection/tests/test_validation.py index aaf4f497f1585..67b66b6a91431 100644 --- a/sklearn/model_selection/tests/test_validation.py +++ b/sklearn/model_selection/tests/test_validation.py @@ -214,6 +214,9 @@ def predict(self, T): T = T.reshape(len(T), -1) return T[:, 0] + def predict_proba(self, T): + return T + def score(self, X=None, Y=None): return 1. / (1 + np.abs(self.a)) @@ -972,6 +975,19 @@ def test_cross_val_predict_unbalanced(): decimal=12) +def test_cross_val_predict_y_none(): + # ensure that cross_val_predict works when y is None + mock_classifier = MockClassifier() + rng = np.random.RandomState(42) + X = rng.rand(100, 10) + y_hat = cross_val_predict(mock_classifier, X, y=None, cv=5, + method='predict') + assert_allclose(X[:, 0], y_hat) + y_hat_proba = cross_val_predict(mock_classifier, X, y=None, cv=5, + method='predict_proba') + assert_allclose(X, y_hat_proba) + + def test_cross_val_score_sparse_fit_params(): iris = load_iris() X, y = iris.data, iris.target @@ -1098,8 +1114,6 @@ def test_learning_curve_incremental_learning_unsupervised(): np.linspace(0.1, 1.0, 10)) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_learning_curve_batch_and_incremental_learning_are_equal(): X, y = make_classification(n_samples=30, n_features=1, n_informative=1, n_redundant=0, n_classes=2, @@ -1167,8 +1181,6 @@ def test_learning_curve_with_boolean_indices(): np.linspace(0.1, 1.0, 10)) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_learning_curve_with_shuffle(): # Following test case was designed this way to verify the code # changes made in pull request: #7506. @@ -1411,7 +1423,6 @@ def test_cross_val_predict_with_method(): LogisticRegression(solver="liblinear")) -@pytest.mark.filterwarnings('ignore: max_iter and tol parameters') def test_cross_val_predict_method_checking(): # Regression test for issue #9639. Tests that cross_val_predict does not # check estimator methods (e.g. predict_proba) before fitting @@ -1637,8 +1648,14 @@ def test_fit_and_score_failing(): "partition for these parameters will be set to %f. " "Details: \n%s" % (fit_and_score_kwargs['error_score'], error_message)) - # check if the same warning is triggered - assert_warns_message(FitFailedWarning, warning_message, _fit_and_score, + + def test_warn_trace(msg): + assert 'Traceback (most recent call last):\n' in msg + split = msg.splitlines() # note: handles more than '\n' + mtb = split[0] + '\n' + split[-1] + return warning_message in mtb + # check traceback is included + assert_warns_message(FitFailedWarning, test_warn_trace, _fit_and_score, *fit_and_score_args, **fit_and_score_kwargs) fit_and_score_kwargs = {'error_score': 'raise'} diff --git a/sklearn/multiclass.py b/sklearn/multiclass.py index 13dda2f6e6927..9eeb4248f83fd 100644 --- a/sklearn/multiclass.py +++ b/sklearn/multiclass.py @@ -52,6 +52,7 @@ check_classification_targets, _ovr_decision_function) from .utils.metaestimators import _safe_split, if_delegate_has_method +from .exceptions import NotFittedError from joblib import Parallel, delayed @@ -433,6 +434,19 @@ def _pairwise(self): def _first_estimator(self): return self.estimators_[0] + @property + def n_features_in_(self): + # For consistency with other estimators we raise a AttributeError so + # that hasattr() fails if the OVR estimator isn't fitted. + try: + check_is_fitted(self) + except NotFittedError as nfe: + raise AttributeError( + "{} object has no n_features_in_ attribute." + .format(self.__class__.__name__) + ) from nfe + return self.estimators_[0].n_features_in_ + def _fit_ovo_binary(estimator, X, y, i, j): """Fit a single binary estimator (one-vs-one).""" @@ -521,7 +535,7 @@ def fit(self, X, y): ------- self """ - X, y = check_X_y(X, y, accept_sparse=['csr', 'csc']) + X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc']) check_classification_targets(y) self.classes_ = np.unique(y) @@ -762,7 +776,7 @@ def fit(self, X, y): ------- self """ - X, y = check_X_y(X, y) + X, y = self._validate_data(X, y) if self.code_size <= 0: raise ValueError("code_size should be greater than 0, got {0}" "".format(self.code_size)) diff --git a/sklearn/multioutput.py b/sklearn/multioutput.py index a6e8fc3c5dc16..82edd85472880 100644 --- a/sklearn/multioutput.py +++ b/sklearn/multioutput.py @@ -24,7 +24,8 @@ from .model_selection import cross_val_predict from .utils import check_array, check_X_y, check_random_state from .utils.metaestimators import if_delegate_has_method -from .utils.validation import check_is_fitted, has_fit_parameter +from .utils.validation import (check_is_fitted, has_fit_parameter, + _check_fit_params) from .utils.multiclass import check_classification_targets from .utils import deprecated @@ -32,12 +33,12 @@ "ClassifierChain", "RegressorChain"] -def _fit_estimator(estimator, X, y, sample_weight=None): +def _fit_estimator(estimator, X, y, sample_weight=None, **fit_params): estimator = clone(estimator) if sample_weight is not None: - estimator.fit(X, y, sample_weight=sample_weight) + estimator.fit(X, y, sample_weight=sample_weight, **fit_params) else: - estimator.fit(X, y) + estimator.fit(X, y, **fit_params) return estimator @@ -121,7 +122,7 @@ def partial_fit(self, X, y, classes=None, sample_weight=None): sample_weight, first_time) for i in range(y.shape[1])) return self - def fit(self, X, y, sample_weight=None): + def fit(self, X, y, sample_weight=None, **fit_params): """ Fit the model to data. Fit a separate model for each output variable. @@ -139,6 +140,9 @@ def fit(self, X, y, sample_weight=None): Only supported if the underlying regressor supports sample weights. + **fit_params : dict of string -> object + Parameters passed to the ``estimator.fit`` method of each step. + Returns ------- self : object @@ -148,9 +152,7 @@ def fit(self, X, y, sample_weight=None): raise ValueError("The base estimator should implement" " a fit method") - X, y = check_X_y(X, y, - multi_output=True, - accept_sparse=True) + X, y = self._validate_data(X, y, multi_output=True, accept_sparse=True) if is_classifier(self): check_classification_targets(y) @@ -164,9 +166,12 @@ def fit(self, X, y, sample_weight=None): raise ValueError("Underlying estimator does not support" " sample weights.") + fit_params_validated = _check_fit_params(X, fit_params) + self.estimators_ = Parallel(n_jobs=self.n_jobs)( delayed(_fit_estimator)( - self.estimator, X, y[:, i], sample_weight) + self.estimator, X, y[:, i], sample_weight, + **fit_params_validated) for i in range(y.shape[1])) return self @@ -258,44 +263,6 @@ def partial_fit(self, X, y, sample_weight=None): super().partial_fit( X, y, sample_weight=sample_weight) - # XXX Remove this method in 0.23 - def score(self, X, y, sample_weight=None): - """Returns the coefficient of determination R^2 of the prediction. - - The coefficient R^2 is defined as (1 - u/v), where u is the residual - sum of squares ((y_true - y_pred) ** 2).sum() and v is the regression - sum of squares ((y_true - y_true.mean()) ** 2).sum(). - Best possible score is 1.0 and it can be negative (because the - model can be arbitrarily worse). A constant model that always - predicts the expected value of y, disregarding the input features, - would get a R^2 score of 0.0. - - Notes - ----- - R^2 is calculated by weighting all the targets equally using - `multioutput='uniform_average'`. - - Parameters - ---------- - X : array-like, shape (n_samples, n_features) - Test samples. - - y : array-like, shape (n_samples) or (n_samples, n_outputs) - True values for X. - - sample_weight : array-like, shape [n_samples], optional - Sample weights. - - Returns - ------- - score : float - R^2 of self.predict(X) wrt. y. - """ - # XXX remove in 0.19 when r2_score default for multioutput changes - from .metrics import r2_score - return r2_score(y, self.predict(X), sample_weight=sample_weight, - multioutput='uniform_average') - class MultiOutputClassifier(ClassifierMixin, _MultiOutputEstimator): """Multi target classification @@ -319,6 +286,9 @@ class MultiOutputClassifier(ClassifierMixin, _MultiOutputEstimator): Attributes ---------- + classes_ : array, shape = (n_classes,) + Class labels. + estimators_ : list of ``n_output`` estimators Estimators used for predictions. @@ -338,7 +308,7 @@ class MultiOutputClassifier(ClassifierMixin, _MultiOutputEstimator): def __init__(self, estimator, n_jobs=None): super().__init__(estimator, n_jobs) - def fit(self, X, Y, sample_weight=None): + def fit(self, X, Y, sample_weight=None, **fit_params): """Fit the model to data matrix X and targets Y. Parameters @@ -351,12 +321,14 @@ def fit(self, X, Y, sample_weight=None): Sample weights. If None, then samples are equally weighted. Only supported if the underlying classifier supports sample weights. + **fit_params : dict of string -> object + Parameters passed to the ``estimator.fit`` method of each step. Returns ------- self : object """ - super().fit(X, Y, sample_weight) + super().fit(X, Y, sample_weight, **fit_params) self.classes_ = [estimator.classes_ for estimator in self.estimators_] return self @@ -433,7 +405,7 @@ def __init__(self, base_estimator, order=None, cv=None, random_state=None): self.random_state = random_state @abstractmethod - def fit(self, X, Y): + def fit(self, X, Y, **fit_params): """Fit the model to data matrix X and targets Y. Parameters @@ -442,12 +414,14 @@ def fit(self, X, Y): The input data. Y : array-like, shape (n_samples, n_classes) The target values. + **fit_params : dict of string -> object + Parameters passed to the `fit` method of each step. Returns ------- self : object """ - X, Y = check_X_y(X, Y, multi_output=True, accept_sparse=True) + X, Y = self._validate_data(X, Y, multi_output=True, accept_sparse=True) random_state = check_random_state(self.random_state) check_array(X, accept_sparse=True) @@ -483,7 +457,8 @@ def fit(self, X, Y): for chain_idx, estimator in enumerate(self.estimators_): y = Y[:, self.order_[chain_idx]] - estimator.fit(X_aug[:, :(X.shape[1] + chain_idx)], y) + estimator.fit(X_aug[:, :(X.shape[1] + chain_idx)], y, + **fit_params) if self.cv is not None and chain_idx < len(self.estimators_) - 1: col_idx = X.shape[1] + chain_idx cv_result = cross_val_predict( @@ -576,12 +551,13 @@ class ClassifierChain(MetaEstimatorMixin, ClassifierMixin, _BaseChain): - An iterable yielding (train, test) splits as arrays of indices. random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - The random number generator is used to generate random chain orders. + If ``order='random'``, determines random number generation for the + chain order. + In addition, it controls the random seed given at each `base_estimator` + at each chaining iteration. Thus, it is only used when `base_estimator` + exposes a `random_state`. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Attributes ---------- @@ -736,12 +712,13 @@ class RegressorChain(MetaEstimatorMixin, RegressorMixin, _BaseChain): - An iterable yielding (train, test) splits as arrays of indices. random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - The random number generator is used to generate random chain orders. + If ``order='random'``, determines random number generation for the + chain order. + In addition, it controls the random seed given at each `base_estimator` + at each chaining iteration. Thus, it is only used when `base_estimator` + exposes a `random_state`. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Attributes ---------- @@ -758,7 +735,8 @@ class RegressorChain(MetaEstimatorMixin, RegressorMixin, _BaseChain): chaining. """ - def fit(self, X, Y): + + def fit(self, X, Y, **fit_params): """Fit the model to data matrix X and targets Y. Parameters @@ -768,11 +746,15 @@ def fit(self, X, Y): Y : array-like, shape (n_samples, n_classes) The target values. + **fit_params : dict of string -> object + Parameters passed to the `fit` method at each step + of the regressor chain. + Returns ------- self : object """ - super().fit(X, Y) + super().fit(X, Y, **fit_params) return self def _more_tags(self): diff --git a/sklearn/naive_bayes.py b/sklearn/naive_bayes.py index 8ebf19125dbf4..bcd9da1cb72fc 100644 --- a/sklearn/naive_bayes.py +++ b/sklearn/naive_bayes.py @@ -45,17 +45,19 @@ def _joint_log_likelihood(self, X): """Compute the unnormalized posterior log probability of X I.e. ``log P(c) + log P(x|c)`` for all rows x of X, as an array-like of - shape [n_classes, n_samples]. + shape (n_classes, n_samples). Input is passed to _joint_log_likelihood as-is by predict, predict_proba and predict_log_proba. """ - @abstractmethod def _check_X(self, X): - """Validate input X - """ - pass + """To be overridden in subclasses with the actual checks.""" + # Note that this is not marked @abstractmethod as long as the + # deprecated public alias sklearn.naive_bayes.BayesNB exists + # (until 0.24) to preserve backward compat for 3rd party projects + # with existing derived classes. + return X def predict(self, X): """ @@ -129,36 +131,33 @@ class GaussianNB(_BaseNB): Parameters ---------- - priors : array-like, shape (n_classes,) + priors : array-like of shape (n_classes,) Prior probabilities of the classes. If specified the priors are not adjusted according to the data. - var_smoothing : float, optional (default=1e-9) + var_smoothing : float, default=1e-9 Portion of the largest variance of all features that is added to variances for calculation stability. Attributes ---------- - class_prior_ : array, shape (n_classes,) - probability of each class. - - class_count_ : array, shape (n_classes,) + class_count_ : ndarray of shape (n_classes,) number of training samples observed in each class. - classes_ : array, shape (n_classes,) - class labels known to the classifier - - theta_ : array, shape (n_classes, n_features) - mean of each feature per class + class_prior_ : ndarray of shape (n_classes,) + probability of each class. - sigma_ : array, shape (n_classes, n_features) - variance of each feature per class + classes_ : ndarray of shape (n_classes,) + class labels known to the classifier epsilon_ : float absolute additive value to variances - classes_ : array-like, shape (n_classes,) - Unique class labels. + sigma_ : ndarray of shape (n_classes, n_features) + variance of each feature per class + + theta_ : ndarray of shape (n_classes, n_features) + mean of each feature per class Examples -------- @@ -187,14 +186,14 @@ def fit(self, X, y, sample_weight=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training vectors, where n_samples is the number of samples and n_features is the number of features. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) Target values. - sample_weight : array-like, shape (n_samples,), optional (default=None) + sample_weight : array-like of shape (n_samples,), default=None Weights applied to individual samples (1. for unweighted). .. versionadded:: 0.17 @@ -204,6 +203,7 @@ def fit(self, X, y, sample_weight=None): ------- self : object """ + X, y = self._validate_data(X, y) y = column_or_1d(y, warn=True) return self._partial_fit(X, y, np.unique(y), _refit=True, sample_weight=sample_weight) @@ -234,21 +234,21 @@ def _update_mean_variance(n_past, mu, var, X, sample_weight=None): weights were given, this should contain the sum of sample weights represented in old mean and variance. - mu : array-like, shape (number of Gaussians,) + mu : array-like of shape (number of Gaussians,) Means for Gaussians in original set. - var : array-like, shape (number of Gaussians,) + var : array-like of shape (number of Gaussians,) Variances for Gaussians in original set. - sample_weight : array-like, shape (n_samples,), optional (default=None) + sample_weight : array-like of shape (n_samples,), default=None Weights applied to individual samples (1. for unweighted). Returns ------- - total_mu : array-like, shape (number of Gaussians,) + total_mu : array-like of shape (number of Gaussians,) Updated mean for each Gaussian over the combined set. - total_var : array-like, shape (number of Gaussians,) + total_var : array-like of shape (number of Gaussians,) Updated variance for each Gaussian over the combined set. """ if X.shape[0] == 0: @@ -302,20 +302,20 @@ def partial_fit(self, X, y, classes=None, sample_weight=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training vectors, where n_samples is the number of samples and n_features is the number of features. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) Target values. - classes : array-like, shape (n_classes,), optional (default=None) + classes : array-like of shape (n_classes,), default=None List of all the classes that can possibly appear in the y vector. Must be provided at the first call to partial_fit, can be omitted in subsequent calls. - sample_weight : array-like, shape (n_samples,), optional (default=None) + sample_weight : array-like of shape (n_samples,), default=None Weights applied to individual samples (1. for unweighted). .. versionadded:: 0.17 @@ -333,24 +333,24 @@ def _partial_fit(self, X, y, classes=None, _refit=False, Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training vectors, where n_samples is the number of samples and n_features is the number of features. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) Target values. - classes : array-like, shape (n_classes,), optional (default=None) + classes : array-like of shape (n_classes,), default=None List of all the classes that can possibly appear in the y vector. Must be provided at the first call to partial_fit, can be omitted in subsequent calls. - _refit : bool, optional (default=False) + _refit : bool, default=False If true, act as though this were the first time we called _partial_fit (ie, throw away any past fitting and start over). - sample_weight : array-like, shape (n_samples,), optional (default=None) + sample_weight : array-like of shape (n_samples,), default=None Weights applied to individual samples (1. for unweighted). Returns @@ -473,7 +473,7 @@ def _check_X(self, X): return check_array(X, accept_sparse='csr') def _check_X_y(self, X, y): - return check_X_y(X, y, accept_sparse='csr') + return self._validate_data(X, y, accept_sparse='csr') def _update_class_log_prior(self, class_prior=None): n_classes = len(self.classes_) @@ -532,7 +532,7 @@ def partial_fit(self, X, y, classes=None, sample_weight=None): y : array-like of shape (n_samples,) Target values. - classes : array-like of shape (n_classes) (default=None) + classes : array-like of shape (n_classes), default=None List of all the classes that can possibly appear in the y vector. Must be provided at the first call to partial_fit, can be omitted @@ -570,8 +570,9 @@ def partial_fit(self, X, y, classes=None, sample_weight=None): # We convert it to np.float64 to support sample_weight consistently Y = Y.astype(np.float64, copy=False) if sample_weight is not None: + sample_weight = _check_sample_weight(sample_weight, X) sample_weight = np.atleast_2d(sample_weight) - Y *= check_array(sample_weight).T + Y *= sample_weight.T class_prior = self.class_prior @@ -622,9 +623,9 @@ def fit(self, X, y, sample_weight=None): # this means we also don't have to cast X to floating point if sample_weight is not None: Y = Y.astype(np.float64, copy=False) - sample_weight = np.asarray(sample_weight) + sample_weight = _check_sample_weight(sample_weight, X) sample_weight = np.atleast_2d(sample_weight) - Y *= check_array(sample_weight).T + Y *= sample_weight.T class_prior = self.class_prior @@ -674,53 +675,50 @@ class MultinomialNB(_BaseDiscreteNB): Parameters ---------- - alpha : float, optional (default=1.0) + alpha : float, default=1.0 Additive (Laplace/Lidstone) smoothing parameter (0 for no smoothing). - fit_prior : boolean, optional (default=True) + fit_prior : bool, default=True Whether to learn class prior probabilities or not. If false, a uniform prior will be used. - class_prior : array-like, size (n_classes,), optional (default=None) + class_prior : array-like of shape (n_classes,), default=None Prior probabilities of the classes. If specified the priors are not adjusted according to the data. Attributes ---------- - class_log_prior_ : array, shape (n_classes, ) - Smoothed empirical log probability for each class. + class_count_ : ndarray of shape (n_classes,) + Number of samples encountered for each class during fitting. This + value is weighted by the sample weight when provided. - intercept_ : array, shape (n_classes, ) - Mirrors ``class_log_prior_`` for interpreting MultinomialNB - as a linear model. + class_log_prior_ : ndarray of shape (n_classes, ) + Smoothed empirical log probability for each class. - feature_log_prob_ : array, shape (n_classes, n_features) - Empirical log probability of features - given a class, ``P(x_i|y)``. + classes_ : ndarray of shape (n_classes,) + Class labels known to the classifier - coef_ : array, shape (n_classes, n_features) + coef_ : ndarray of shape (n_classes, n_features) Mirrors ``feature_log_prob_`` for interpreting MultinomialNB as a linear model. - class_count_ : array, shape (n_classes,) - Number of samples encountered for each class during fitting. This - value is weighted by the sample weight when provided. - - classes_ : array, shape (n_classes,) - Class labels known to the classifier - - feature_count_ : array, shape (n_classes, n_features) + feature_count_ : ndarray of shape (n_classes, n_features) Number of samples encountered for each (class, feature) during fitting. This value is weighted by the sample weight when provided. + feature_log_prob_ : ndarray of shape (n_classes, n_features) + Empirical log probability of features + given a class, ``P(x_i|y)``. + + intercept_ : ndarray of shape (n_classes, ) + Mirrors ``class_log_prior_`` for interpreting MultinomialNB + as a linear model. + n_features_ : int Number of features of each sample. - classes_ : array-like, shape (n_classes,) - Unique class labels. - Examples -------- >>> import numpy as np @@ -786,16 +784,16 @@ class ComplementNB(_BaseDiscreteNB): Parameters ---------- - alpha : float, optional (default=1.0) + alpha : float, default=1.0 Additive (Laplace/Lidstone) smoothing parameter (0 for no smoothing). - fit_prior : boolean, optional (default=True) + fit_prior : bool, default=True Only used in edge case with a single class in the training set. - class_prior : array-like, size (n_classes,), optional (default=None) + class_prior : array-like of shape (n_classes,), default=None Prior probabilities of the classes. Not used. - norm : boolean, optional (default=False) + norm : bool, default=False Whether or not a second normalization of the weights is performed. The default behavior mirrors the implementations found in Mahout and Weka, which do not follow the full algorithm described in Table 9 of the @@ -803,34 +801,31 @@ class ComplementNB(_BaseDiscreteNB): Attributes ---------- - class_log_prior_ : array, shape (n_classes, ) + class_count_ : ndarray of shape (n_classes,) + Number of samples encountered for each class during fitting. This + value is weighted by the sample weight when provided. + + class_log_prior_ : ndarray of shape (n_classes,) Smoothed empirical log probability for each class. Only used in edge case with a single class in the training set. - feature_log_prob_ : array, shape (n_classes, n_features) - Empirical weights for class complements. + classes_ : ndarray of shape (n_classes,) + Class labels known to the classifier - class_count_ : array, shape (n_classes,) - Number of samples encountered for each class during fitting. This + feature_all_ : ndarray of shape (n_features,) + Number of samples encountered for each feature during fitting. This value is weighted by the sample weight when provided. - classes_ : array, shape (n_classes,) - Class labels known to the classifier - - feature_count_ : array, shape (n_classes, n_features) + feature_count_ : ndarray of shape (n_classes, n_features) Number of samples encountered for each (class, feature) during fitting. This value is weighted by the sample weight when provided. + feature_log_prob_ : ndarray of shape (n_classes, n_features) + Empirical weights for class complements. + n_features_ : int Number of features of each sample. - feature_all_ : array, shape (n_features,) - Number of samples encountered for each feature during fitting. This - value is weighted by the sample weight when provided. - - classes_ : array of shape (n_classes,) - The classes labels. - Examples -------- >>> import numpy as np @@ -900,52 +895,57 @@ class BernoulliNB(_BaseDiscreteNB): Parameters ---------- - alpha : float, optional (default=1.0) + alpha : float, default=1.0 Additive (Laplace/Lidstone) smoothing parameter (0 for no smoothing). - binarize : float or None, optional (default=0.0) + binarize : float or None, default=0.0 Threshold for binarizing (mapping to booleans) of sample features. If None, input is presumed to already consist of binary vectors. - fit_prior : bool, optional (default=True) + fit_prior : bool, default=True Whether to learn class prior probabilities or not. If false, a uniform prior will be used. - class_prior : array-like, size=[n_classes,], optional (default=None) + class_prior : array-like of shape (n_classes,), default=None Prior probabilities of the classes. If specified the priors are not adjusted according to the data. Attributes ---------- - class_log_prior_ : array, shape = [n_classes] - Log probability of each class (smoothed). - - feature_log_prob_ : array, shape = [n_classes, n_features] - Empirical log probability of features given a class, P(x_i|y). - - class_count_ : array, shape = [n_classes] + class_count_ : ndarray of shape (n_classes) Number of samples encountered for each class during fitting. This value is weighted by the sample weight when provided. - classes_ : array, shape (n_classes,) + class_log_prior_ : ndarray of shape (n_classes) + Log probability of each class (smoothed). + + classes_ : ndarray of shape (n_classes,) Class labels known to the classifier - feature_count_ : array, shape = [n_classes, n_features] + feature_count_ : ndarray of shape (n_classes, n_features) Number of samples encountered for each (class, feature) during fitting. This value is weighted by the sample weight when provided. + feature_log_prob_ : ndarray of shape (n_classes, n_features) + Empirical log probability of features given a class, P(x_i|y). + n_features_ : int Number of features of each sample. - classes_ : array of shape (n_classes,) - The classes labels. - - See Also - ---------- - MultinomialNB: The multinomial Naive Bayes classifier is \ - suitable for classification with discrete features. + Examples + -------- + >>> import numpy as np + >>> rng = np.random.RandomState(1) + >>> X = rng.randint(5, size=(6, 100)) + >>> Y = np.array([1, 2, 3, 4, 4, 5]) + >>> from sklearn.naive_bayes import BernoulliNB + >>> clf = BernoulliNB() + >>> clf.fit(X, Y) + BernoulliNB() + >>> print(clf.predict(X[2:3])) + [3] References ---------- @@ -959,19 +959,6 @@ class BernoulliNB(_BaseDiscreteNB): V. Metsis, I. Androutsopoulos and G. Paliouras (2006). Spam filtering with naive Bayes -- Which naive Bayes? 3rd Conf. on Email and Anti-Spam (CEAS). - - Examples - -------- - >>> import numpy as np - >>> rng = np.random.RandomState(1) - >>> X = rng.randint(5, size=(6, 100)) - >>> Y = np.array([1, 2, 3, 4, 4, 5]) - >>> from sklearn.naive_bayes import BernoulliNB - >>> clf = BernoulliNB() - >>> clf.fit(X, Y) - BernoulliNB() - >>> print(clf.predict(X[2:3])) - [3] """ def __init__(self, alpha=1.0, binarize=.0, fit_prior=True, @@ -1034,36 +1021,39 @@ class CategoricalNB(_BaseDiscreteNB): Parameters ---------- - alpha : float, optional (default=1.0) + alpha : float, default=1.0 Additive (Laplace/Lidstone) smoothing parameter (0 for no smoothing). - fit_prior : boolean, optional (default=True) + fit_prior : bool, default=True Whether to learn class prior probabilities or not. If false, a uniform prior will be used. - class_prior : array-like, size (n_classes,), optional (default=None) + class_prior : array-like of shape (n_classes,), default=None Prior probabilities of the classes. If specified the priors are not adjusted according to the data. Attributes ---------- - class_log_prior_ : array, shape (n_classes, ) - Smoothed empirical log probability for each class. - - feature_log_prob_ : list of arrays, len n_features + category_count_ : list of arrays of shape (n_features,) Holds arrays of shape (n_classes, n_categories of respective feature) - for each feature. Each array provides the empirical log probability - of categories given the respective feature and class, ``P(x_i|y)``. + for each feature. Each array provides the number of samples + encountered for each class and category of the specific feature. - class_count_ : array, shape (n_classes,) + class_count_ : ndarray of shape (n_classes,) Number of samples encountered for each class during fitting. This value is weighted by the sample weight when provided. - category_count_ : list of arrays, len n_features + class_log_prior_ : ndarray of shape (n_classes,) + Smoothed empirical log probability for each class. + + classes_ : ndarray of shape (n_classes,) + Class labels known to the classifier + + feature_log_prob_ : list of arrays of shape (n_features,) Holds arrays of shape (n_classes, n_categories of respective feature) - for each feature. Each array provides the number of samples - encountered for each class and category of the specific feature. + for each feature. Each array provides the empirical log probability + of categories given the respective feature and class, ``P(x_i|y)``. n_features_ : int Number of features of each sample. @@ -1092,7 +1082,7 @@ def fit(self, X, y, sample_weight=None): Parameters ---------- - X : {array-like, sparse matrix}, shape = [n_samples, n_features] + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training vectors, where n_samples is the number of samples and n_features is the number of features. Here, each feature of X is assumed to be from a different categorical distribution. @@ -1101,10 +1091,10 @@ def fit(self, X, y, sample_weight=None): total number of categories for the given feature. This can, for instance, be achieved with the help of OrdinalEncoder. - y : array-like, shape = [n_samples] + y : array-like of shape (n_samples,) Target values. - sample_weight : array-like, shape = [n_samples], (default=None) + sample_weight : array-like of shape (n_samples), default=None Weights applied to individual samples (1. for unweighted). Returns @@ -1129,7 +1119,7 @@ def partial_fit(self, X, y, classes=None, sample_weight=None): Parameters ---------- - X : {array-like, sparse matrix}, shape = [n_samples, n_features] + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training vectors, where n_samples is the number of samples and n_features is the number of features. Here, each feature of X is assumed to be from a different categorical distribution. @@ -1138,16 +1128,16 @@ def partial_fit(self, X, y, classes=None, sample_weight=None): total number of categories for the given feature. This can, for instance, be achieved with the help of OrdinalEncoder. - y : array-like, shape = [n_samples] + y : array-like of shape (n_samples) Target values. - classes : array-like, shape = [n_classes] (default=None) + classes : array-like of shape (n_classes), default=None List of all the classes that can possibly appear in the y vector. Must be provided at the first call to partial_fit, can be omitted in subsequent calls. - sample_weight : array-like, shape = [n_samples], (default=None) + sample_weight : array-like of shape (n_samples), default=None Weights applied to individual samples (1. for unweighted). Returns @@ -1157,24 +1147,19 @@ def partial_fit(self, X, y, classes=None, sample_weight=None): return super().partial_fit(X, y, classes, sample_weight=sample_weight) + def _more_tags(self): + return {'requires_positive_X': True} + def _check_X(self, X): - # FIXME: we can avoid calling check_array twice after #14872 is merged. - # X = check_array(X, y, dtype='int', accept_sparse=False, - # force_all_finite=True) - X = check_array(X, accept_sparse=False, force_all_finite=True) - X = check_array(X, dtype='int') - if np.any(X < 0): - raise ValueError("X must not contain negative values.") + X = check_array(X, dtype='int', accept_sparse=False, + force_all_finite=True) + check_non_negative(X, "CategoricalNB (input X)") return X def _check_X_y(self, X, y): - # FIXME: we can avoid calling check_array twice after #14872 is merged. - # X, y = check_array(X, y, dtype='int', accept_sparse=False, - # force_all_finite=True) - X, y = check_X_y(X, y, accept_sparse=False, force_all_finite=True) - X, y = check_X_y(X, y, dtype='int') - if np.any(X < 0): - raise ValueError("X must not contain negative values.") + X, y = self._validate_data(X, y, dtype='int', accept_sparse=False, + force_all_finite=True) + check_non_negative(X, "CategoricalNB (input X)") return X, y def _init_counters(self, n_effective_classes, n_features): @@ -1223,7 +1208,7 @@ def _update_feature_log_prob(self, alpha): def _joint_log_likelihood(self, X): if not X.shape[1] == self.n_features_: raise ValueError("Expected input with %d features, got %d instead" - .format(self.n_features_, X.shape[1])) + % (self.n_features_, X.shape[1])) jll = np.zeros((X.shape[0], self.class_count_.shape[0])) for i in range(self.n_features_): indices = X[:, i] diff --git a/sklearn/neighbors/_base.py b/sklearn/neighbors/_base.py index ef97b9df93718..945959ef10d9c 100644 --- a/sklearn/neighbors/_base.py +++ b/sklearn/neighbors/_base.py @@ -24,6 +24,7 @@ from ..metrics import pairwise_distances_chunked from ..metrics.pairwise import PAIRWISE_DISTANCE_FUNCTIONS from ..utils import check_X_y, check_array, gen_even_slices +from ..utils import _to_object_array from ..utils.multiclass import check_classification_targets from ..utils.validation import check_is_fitted from ..utils.validation import check_non_negative @@ -262,7 +263,7 @@ def _radius_neighbors_from_graph(graph, radius, return_distance): """ assert graph.format == 'csr' - no_filter_needed = graph.data.max() <= radius + no_filter_needed = bool(graph.data.max() <= radius) if no_filter_needed: data, indices, indptr = graph.data, graph.indices, graph.indptr @@ -276,8 +277,8 @@ def _radius_neighbors_from_graph(graph, radius, return_distance): indices = indices.astype(np.intp, copy=no_filter_needed) if return_distance: - neigh_dist = np.array(np.split(data, indptr[1:-1])) - neigh_ind = np.array(np.split(indices, indptr[1:-1])) + neigh_dist = _to_object_array(np.split(data, indptr[1:-1])) + neigh_ind = _to_object_array(np.split(indices, indptr[1:-1])) if return_distance: return neigh_dist, neigh_ind @@ -395,8 +396,9 @@ def _fit(self, X): if self.effective_metric_ == 'precomputed': X = _check_precomputed(X) + self.n_features_in_ = X.shape[1] else: - X = check_array(X, accept_sparse='csr') + X = self._validate_data(X, accept_sparse='csr') n_samples = X.shape[0] if n_samples == 0: @@ -558,7 +560,7 @@ def kneighbors(self, X=None, n_neighbors=None, return_distance=True): Examples -------- - In the following example, we construct a NeighborsClassifier + In the following example, we construct a NearestNeighbors class from an array representing our data set and ask who's the closest point to [1,1,1] @@ -940,17 +942,12 @@ class from an array representing our data set and ask who's neigh_dist_chunks, neigh_ind_chunks = zip(*chunked_results) neigh_dist_list = sum(neigh_dist_chunks, []) neigh_ind_list = sum(neigh_ind_chunks, []) - # See https://github.com/numpy/numpy/issues/5456 - # to understand why this is initialized this way. - neigh_dist = np.empty(len(neigh_dist_list), dtype='object') - neigh_dist[:] = neigh_dist_list - neigh_ind = np.empty(len(neigh_ind_list), dtype='object') - neigh_ind[:] = neigh_ind_list + neigh_dist = _to_object_array(neigh_dist_list) + neigh_ind = _to_object_array(neigh_ind_list) results = neigh_dist, neigh_ind else: neigh_ind_list = sum(chunked_results, []) - results = np.empty(len(neigh_ind_list), dtype='object') - results[:] = neigh_ind_list + results = _to_object_array(neigh_ind_list) elif self._fit_method in ['ball_tree', 'kd_tree']: if issparse(X): diff --git a/sklearn/neighbors/_binary_tree.pxi b/sklearn/neighbors/_binary_tree.pxi index 43db83b0f8b62..ef6a2a2d5d330 100755 --- a/sklearn/neighbors/_binary_tree.pxi +++ b/sklearn/neighbors/_binary_tree.pxi @@ -180,18 +180,8 @@ cdef struct NodeHeapData_t: ITYPE_t i2 # build the corresponding numpy dtype for NodeHeapData -# There is no offsetof() function in cython, so we hack it. -# If we can ensure numpy 1.5 or greater, a cleaner way is to do -# cdef NodeHeapData_t nhd_tmp -# NodeHeapData = np.asarray((&nhd_tmp)).dtype cdef NodeHeapData_t nhd_tmp -offsets = [&(nhd_tmp.val) - &nhd_tmp, - &(nhd_tmp.i1) - &nhd_tmp, - &(nhd_tmp.i2) - &nhd_tmp] -NodeHeapData = np.dtype({'names': ['val', 'i1', 'i2'], - 'formats': [DTYPE, ITYPE, ITYPE], - 'offsets': offsets, - 'itemsize': sizeof(NodeHeapData_t)}) +NodeHeapData = np.asarray((&nhd_tmp)).dtype cdef struct NodeData_t: ITYPE_t idx_start @@ -200,19 +190,8 @@ cdef struct NodeData_t: DTYPE_t radius # build the corresponding numpy dtype for NodeData -# There is no offsetof() function in cython, so we hack it. -# If we can ensure numpy 1.5 or greater, a cleaner way is to do -# cdef NodeData_t nd_tmp -# NodeData = np.asarray((&nd_tmp)).dtype cdef NodeData_t nd_tmp -offsets = [&(nd_tmp.idx_start) - &nd_tmp, - &(nd_tmp.idx_end) - &nd_tmp, - &(nd_tmp.is_leaf) - &nd_tmp, - &(nd_tmp.radius) - &nd_tmp] -NodeData = np.dtype({'names': ['idx_start', 'idx_end', 'is_leaf', 'radius'], - 'formats': [ITYPE, ITYPE, ITYPE, DTYPE], - 'offsets': offsets, - 'itemsize': sizeof(NodeData_t)}) +NodeData = np.asarray((&nd_tmp)).dtype ###################################################################### @@ -272,7 +251,7 @@ X : array-like of shape (n_samples, n_features) Note: if X is a C-contiguous array of doubles then data will not be copied. Otherwise, an internal copy will be made. -leaf_size : positive integer (default = 40) +leaf_size : positive int, default=40 Number of points at which to switch to brute-force. Changing leaf_size will not affect the results of a query, but can significantly impact the speed of a query and the memory required @@ -282,7 +261,7 @@ leaf_size : positive integer (default = 40) satisfy ``leaf_size <= n_points <= 2 * leaf_size``, except in the case that ``n_samples < leaf_size``. -metric : string or DistanceMetric object +metric : str or DistanceMetric object the distance metric to use for the tree. Default='minkowski' with p=2 (that is, a euclidean metric). See the documentation of the DistanceMetric class for a list of available metrics. @@ -509,15 +488,15 @@ def kernel_norm(h, d, kernel, return_log=False): Parameters ---------- h : float - the bandwidth of the kernel + The bandwidth of the kernel. d : int - the dimension of the space in which the kernel norm is computed - kernel : string + The dimension of the space in which the kernel norm is computed. + kernel : str The kernel identifier. Must be one of ['gaussian'|'tophat'|'epanechnikov'| 'exponential'|'linear'|'cosine'] - return_log : boolean - if True, return the log of the kernel norm. Otherwise, return the + return_log : bool, default=False + If True, return the log of the kernel norm. Otherwise, return the kernel norm. Returns ------- @@ -1281,20 +1260,20 @@ cdef class BinaryTree: ---------- X : array-like of shape (n_samples, n_features) An array of points to query - k : integer (default = 1) + k : int, default=1 The number of nearest neighbors to return - return_distance : boolean (default = True) + return_distance : bool, default=True if True, return a tuple (d, i) of distances and indices if False, return array i - dualtree : boolean (default = False) + dualtree : bool, default=False if True, use the dual tree formalism for the query: a tree is built for the query points, and the pair of trees is used to efficiently search this space. This can lead to better performance as the number of points grows large. - breadth_first : boolean (default = False) + breadth_first : bool, default=False if True, then query the nodes in a breadth-first manner. Otherwise, query the nodes in a depth-first manner. - sort_results : boolean (default = True) + sort_results : bool, default=True if True, then distances and indices of each point are sorted on return, so that the first column contains the closest points. Otherwise, neighbors are returned in an arbitrary order. @@ -1304,13 +1283,13 @@ cdef class BinaryTree: i : if return_distance == False (d,i) : if return_distance == True - d : array of doubles - shape: x.shape[:-1] + (k,) - each entry gives the list of distances to the - neighbors of the corresponding point + d : ndarray of shape X.shape[:-1] + k, dtype=double + Each entry gives the list of distances to the neighbors of the + corresponding point. - i : array of integers - shape: x.shape[:-1] + (k,) - each entry gives the list of indices of - neighbors of the corresponding point + i : ndarray of shape X.shape[:-1] + k, dtype=int + Each entry gives the list of indices of neighbors of the + corresponding point. """ # XXX: we should allow X to be a pre-built tree. X = check_array(X, dtype=DTYPE, order='C') @@ -1394,19 +1373,19 @@ cdef class BinaryTree: r : distance within which neighbors are returned r can be a single value, or an array of values of shape x.shape[:-1] if different radii are desired for each point. - return_distance : boolean (default = False) + return_distance : bool, default=False if True, return distances to neighbors of each point if False, return only neighbors Note that unlike the query() method, setting return_distance=True here adds to the computation time. Not all distances need to be calculated explicitly for return_distance=False. Results are not sorted by default: see ``sort_results`` keyword. - count_only : boolean (default = False) + count_only : bool, default=False if True, return only the count of points within distance r if False, return the indices of all points within distance r If return_distance==True, setting count_only=True will result in an error. - sort_results : boolean (default = False) + sort_results : bool, default=False if True, the distances and indices will be sorted before being returned. If False, the results will not be sorted. If return_distance == False, setting sort_results = True will @@ -1418,19 +1397,19 @@ cdef class BinaryTree: ind : if count_only == False and return_distance == False (ind, dist) : if count_only == False and return_distance == True - count : array of integers, shape = X.shape[:-1] - each entry gives the number of neighbors within - a distance r of the corresponding point. + count : ndarray of shape X.shape[:-1], dtype=int + Each entry gives the number of neighbors within a distance r of the + corresponding point. - ind : array of objects, shape = X.shape[:-1] - each element is a numpy integer array listing the indices of + ind : ndarray of shape X.shape[:-1], dtype=object + Each element is a numpy integer array listing the indices of neighbors of the corresponding point. Note that unlike the results of a k-neighbors query, the returned neighbors are not sorted by distance by default. - dist : array of objects, shape = X.shape[:-1] - each element is a numpy double array - listing the distances corresponding to indices in i. + dist : ndarray of shape X.shape[:-1], dtype=object + Each element is a numpy double array listing the distances + corresponding to indices in i. """ if count_only and return_distance: raise ValueError("count_only and return_distance " @@ -1591,7 +1570,7 @@ cdef class BinaryTree: of training data. h : float the bandwidth of the kernel - kernel : string + kernel : str, default="gaussian" specify the kernel to use. Options are - 'gaussian' - 'tophat' @@ -1600,23 +1579,23 @@ cdef class BinaryTree: - 'linear' - 'cosine' Default is kernel = 'gaussian' - atol, rtol : float (default = 0) + atol, rtol : float, default=0, 1e-8 Specify the desired relative and absolute tolerance of the result. If the true result is K_true, then the returned result K_ret satisfies ``abs(K_true - K_ret) < atol + rtol * K_ret`` The default is zero (i.e. machine precision) for both. - breadth_first : boolean (default = False) - if True, use a breadth-first search. If False (default) use a + breadth_first : bool, default=False + If True, use a breadth-first search. If False (default) use a depth-first search. Breadth-first is generally faster for compact kernels and/or high tolerances. - return_log : boolean (default = False) - return the logarithm of the result. This can be more accurate + return_log : bool, default=False + Return the logarithm of the result. This can be more accurate than returning the result itself for narrow kernels. Returns ------- - density : ndarray - The array of (log)-density evaluations, shape = X.shape[:-1] + density : ndarray of shape X.shape[:-1] + The array of (log)-density evaluations """ cdef DTYPE_t h_c = h cdef DTYPE_t log_atol = log(atol) @@ -1722,10 +1701,10 @@ cdef class BinaryTree: X : array-like of shape (n_samples, n_features) An array of points to query. Last dimension should match dimension of training data. - r : array_like + r : array-like A one-dimensional array of distances - dualtree : boolean (default = False) - If true, use a dualtree algorithm. Otherwise, use a single-tree + dualtree : bool, default=False + If True, use a dualtree algorithm. Otherwise, use a single-tree algorithm. Dual tree algorithms can have better scaling for large N. diff --git a/sklearn/neighbors/_classification.py b/sklearn/neighbors/_classification.py index af3a9feb857e5..0580b710afd44 100644 --- a/sklearn/neighbors/_classification.py +++ b/sklearn/neighbors/_classification.py @@ -30,10 +30,10 @@ class KNeighborsClassifier(NeighborsBase, KNeighborsMixin, Parameters ---------- - n_neighbors : int, optional (default = 5) + n_neighbors : int, default=5 Number of neighbors to use by default for :meth:`kneighbors` queries. - weights : str or callable, optional (default = 'uniform') + weights : {'uniform', 'distance'} or callable, default='uniform' weight function used in prediction. Possible values: - 'uniform' : uniform weights. All points in each neighborhood @@ -45,7 +45,7 @@ class KNeighborsClassifier(NeighborsBase, KNeighborsMixin, array of distances, and returns an array of the same shape containing the weights. - algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional + algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto' Algorithm used to compute the nearest neighbors: - 'ball_tree' will use :class:`BallTree` @@ -57,18 +57,18 @@ class KNeighborsClassifier(NeighborsBase, KNeighborsMixin, Note: fitting on sparse input will override the setting of this parameter, using brute force. - leaf_size : int, optional (default = 30) + leaf_size : int, default=30 Leaf size passed to BallTree or KDTree. This can affect the speed of the construction and query, as well as the memory required to store the tree. The optimal value depends on the nature of the problem. - p : integer, optional (default = 2) + p : int, default=2 Power parameter for the Minkowski metric. When p = 1, this is equivalent to using manhattan_distance (l1), and euclidean_distance (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used. - metric : string or callable, default 'minkowski' + metric : str or callable, default='minkowski' the distance metric to use for the tree. The default metric is minkowski, and with p=2 is equivalent to the standard Euclidean metric. See the documentation of the DistanceMetric class for a @@ -77,10 +77,10 @@ class KNeighborsClassifier(NeighborsBase, KNeighborsMixin, must be square during fit. X may be a :term:`Glossary `, in which case only "nonzero" elements may be considered neighbors. - metric_params : dict, optional (default = None) + metric_params : dict, default=None Additional keyword arguments for the metric function. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of parallel jobs to run for neighbors search. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` @@ -92,7 +92,7 @@ class KNeighborsClassifier(NeighborsBase, KNeighborsMixin, classes_ : array of shape (n_classes,) Class labels known to the classifier - effective_metric_ : string or callble + effective_metric_ : str or callble The distance metric used. It will be same as the `metric` parameter or a synonym of it, e.g. 'euclidean' if the `metric` parameter set to 'minkowski' and `p` parameter set to 2. @@ -159,13 +159,13 @@ def predict(self, X): Parameters ---------- - X : array-like, shape (n_queries, n_features), \ + X : array-like of shape (n_queries, n_features), \ or (n_queries, n_indexed) if metric == 'precomputed' Test samples. Returns ------- - y : array of shape [n_queries] or [n_queries, n_outputs] + y : ndarray of shape (n_queries,) or (n_queries, n_outputs) Class labels for each data sample. """ X = check_array(X, accept_sparse='csr') @@ -201,13 +201,13 @@ def predict_proba(self, X): Parameters ---------- - X : array-like, shape (n_queries, n_features), \ + X : array-like of shape (n_queries, n_features), \ or (n_queries, n_indexed) if metric == 'precomputed' Test samples. Returns ------- - p : array of shape = [n_queries, n_classes], or a list of n_outputs + p : ndarray of shape (n_queries, n_classes), or a list of n_outputs of such arrays if n_outputs > 1. The class probabilities of the input samples. Classes are ordered by lexicographic order. @@ -259,11 +259,11 @@ class RadiusNeighborsClassifier(NeighborsBase, RadiusNeighborsMixin, Parameters ---------- - radius : float, optional (default = 1.0) + radius : float, default=1.0 Range of parameter space to use by default for :meth:`radius_neighbors` queries. - weights : str or callable + weights : {'uniform', 'distance'} or callable, default='uniform' weight function used in prediction. Possible values: - 'uniform' : uniform weights. All points in each neighborhood @@ -277,7 +277,7 @@ class RadiusNeighborsClassifier(NeighborsBase, RadiusNeighborsMixin, Uniform weights are used by default. - algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional + algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto' Algorithm used to compute the nearest neighbors: - 'ball_tree' will use :class:`BallTree` @@ -289,18 +289,18 @@ class RadiusNeighborsClassifier(NeighborsBase, RadiusNeighborsMixin, Note: fitting on sparse input will override the setting of this parameter, using brute force. - leaf_size : int, optional (default = 30) + leaf_size : int, default=30 Leaf size passed to BallTree or KDTree. This can affect the speed of the construction and query, as well as the memory required to store the tree. The optimal value depends on the nature of the problem. - p : integer, optional (default = 2) + p : int, default=2 Power parameter for the Minkowski metric. When p = 1, this is equivalent to using manhattan_distance (l1), and euclidean_distance (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used. - metric : string or callable, default 'minkowski' + metric : str or callable, default='minkowski' the distance metric to use for the tree. The default metric is minkowski, and with p=2 is equivalent to the standard Euclidean metric. See the documentation of the DistanceMetric class for a @@ -309,7 +309,7 @@ class RadiusNeighborsClassifier(NeighborsBase, RadiusNeighborsMixin, must be square during fit. X may be a :term:`Glossary `, in which case only "nonzero" elements may be considered neighbors. - outlier_label : {manual label, 'most_frequent'}, optional (default = None) + outlier_label : {manual label, 'most_frequent'}, default=None label for outlier samples (samples with no neighbors in given radius). - manual label: str or int label (should be the same type as y) @@ -317,10 +317,10 @@ class RadiusNeighborsClassifier(NeighborsBase, RadiusNeighborsMixin, - 'most_frequent' : assign the most frequent label of y to outliers. - None : when any outlier is detected, ValueError will be raised. - metric_params : dict, optional (default = None) + metric_params : dict, default=None Additional keyword arguments for the metric function. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of parallel jobs to run for neighbors search. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` @@ -328,10 +328,10 @@ class RadiusNeighborsClassifier(NeighborsBase, RadiusNeighborsMixin, Attributes ---------- - classes_ : array of shape (n_classes,) + classes_ : ndarray of shape (n_classes,) Class labels known to the classifier. - effective_metric_ : string or callble + effective_metric_ : str or callble The distance metric used. It will be same as the `metric` parameter or a synonym of it, e.g. 'euclidean' if the `metric` parameter set to 'minkowski' and `p` parameter set to 2. @@ -392,12 +392,14 @@ def fit(self, X, y): Parameters ---------- - X : {array-like, sparse matrix, BallTree, KDTree} - Training data. If array or matrix, shape [n_samples, n_features], - or [n_samples, n_samples] if metric='precomputed'. + X : BallTree, KDTree or {array-like, sparse matrix} of shape \ + (n_samples, n_features) or (n_samples, n_samples) + Training data. If array or matrix, the shape is (n_samples, + n_features), or (n_samples, n_samples) if metric='precomputed'. - y : {array-like, sparse matrix} - Target values of shape = [n_samples] or [n_samples, n_outputs] + y : {array-like, sparse matrix} of shape (n_samples,) or \ + (n_samples, n_output) + Target values. """ @@ -453,13 +455,13 @@ def predict(self, X): Parameters ---------- - X : array-like, shape (n_queries, n_features), \ + X : array-like of shape (n_queries, n_features), \ or (n_queries, n_indexed) if metric == 'precomputed' Test samples. Returns ------- - y : array of shape [n_queries] or [n_queries, n_outputs] + y : ndarray of shape (n_queries,) or (n_queries, n_outputs) Class labels for each data sample. """ @@ -495,13 +497,13 @@ def predict_proba(self, X): Parameters ---------- - X : array-like, shape (n_queries, n_features), \ + X : array-like of shape (n_queries, n_features), \ or (n_queries, n_indexed) if metric == 'precomputed' Test samples. Returns ------- - p : array of shape = [n_queries, n_classes], or a list of n_outputs + p : ndarray of shape (n_queries, n_classes), or a list of n_outputs of such arrays if n_outputs > 1. The class probabilities of the input samples. Classes are ordered by lexicographic order. diff --git a/sklearn/neighbors/_graph.py b/sklearn/neighbors/_graph.py index 81616fbf3651b..9fc4a6e830cde 100644 --- a/sklearn/neighbors/_graph.py +++ b/sklearn/neighbors/_graph.py @@ -52,23 +52,23 @@ def kneighbors_graph(X, n_neighbors, mode='connectivity', metric='minkowski', n_neighbors : int Number of neighbors for each sample. - mode : {'connectivity', 'distance'}, optional + mode : {'connectivity', 'distance'}, default='connectivity' Type of returned matrix: 'connectivity' will return the connectivity matrix with ones and zeros, and 'distance' will return the distances between neighbors according to the given metric. - metric : string, default 'minkowski' + metric : str, default='minkowski' The distance metric used to calculate the k-Neighbors for each sample point. The DistanceMetric class gives a list of available metrics. The default distance is 'euclidean' ('minkowski' metric with the p param equal to 2.) - p : int, default 2 + p : int, default=2 Power parameter for the Minkowski metric. When p = 1, this is equivalent to using manhattan_distance (l1), and euclidean_distance (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used. - metric_params : dict, optional + metric_params : dict, default=None additional keyword arguments for the metric function. include_self : bool or 'auto', default=False @@ -76,7 +76,7 @@ def kneighbors_graph(X, n_neighbors, mode='connectivity', metric='minkowski', itself. If 'auto', then True is used for mode='connectivity' and False for mode='distance'. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of parallel jobs to run for neighbors search. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` @@ -84,8 +84,9 @@ def kneighbors_graph(X, n_neighbors, mode='connectivity', metric='minkowski', Returns ------- - A : sparse graph in CSR format, shape = [n_samples, n_samples] - A[i, j] is assigned the weight of edge that connects i to j. + A : sparse matrix of shape (n_samples, n_samples) + Graph where A[i, j] is assigned the weight of edge that + connects i to j. The matrix is of CSR format. Examples -------- @@ -130,23 +131,23 @@ def radius_neighbors_graph(X, radius, mode='connectivity', metric='minkowski', radius : float Radius of neighborhoods. - mode : {'connectivity', 'distance'}, optional + mode : {'connectivity', 'distance'}, default='connectivity' Type of returned matrix: 'connectivity' will return the connectivity matrix with ones and zeros, and 'distance' will return the distances between neighbors according to the given metric. - metric : string, default 'minkowski' + metric : str, default='minkowski' The distance metric used to calculate the neighbors within a given radius for each sample point. The DistanceMetric class gives a list of available metrics. The default distance is 'euclidean' ('minkowski' metric with the param equal to 2.) - p : int, default 2 + p : int, default=2 Power parameter for the Minkowski metric. When p = 1, this is equivalent to using manhattan_distance (l1), and euclidean_distance (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used. - metric_params : dict, optional + metric_params : dict, default=None additional keyword arguments for the metric function. include_self : bool or 'auto', default=False @@ -154,7 +155,7 @@ def radius_neighbors_graph(X, radius, mode='connectivity', metric='minkowski', itself. If 'auto', then True is used for mode='connectivity' and False for mode='distance'. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of parallel jobs to run for neighbors search. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` @@ -162,8 +163,9 @@ def radius_neighbors_graph(X, radius, mode='connectivity', metric='minkowski', Returns ------- - A : sparse graph in CSR format, shape = [n_samples, n_samples] - A[i, j] is assigned the weight of edge that connects i to j. + A : sparse matrix of shape (n_samples, n_samples) + Graph where A[i, j] is assigned the weight of edge that connects + i to j. The matrix is of CSR format. Examples -------- @@ -231,7 +233,7 @@ class KNeighborsTransformer(NeighborsBase, KNeighborsMixin, required to store the tree. The optimal value depends on the nature of the problem. - metric : string or callable, default='minkowski' + metric : str or callable, default='minkowski' metric to use for distance computation. Any metric from scikit-learn or scipy.spatial.distance can be used. @@ -294,14 +296,15 @@ def transform(self, X): Parameters ---------- X : array-like of shape (n_samples_transform, n_features) - Sample data + Sample data. Returns ------- - Xt : CSR sparse graph of shape (n_samples_transform, n_samples_fit) + Xt : sparse matrix of shape (n_samples_transform, n_samples_fit) Xt[i, j] is assigned the weight of edge that connects i to j. Only the neighbors have an explicit value. The diagonal is always explicit. + The matrix is of CSR format. """ check_is_fitted(self) add_one = self.mode == 'distance' @@ -323,10 +326,11 @@ def fit_transform(self, X, y=None): Returns ------- - Xt : CSR sparse graph of shape (n_samples, n_samples) + Xt : sparse matrix of shape (n_samples, n_samples) Xt[i, j] is assigned the weight of edge that connects i to j. Only the neighbors have an explicit value. The diagonal is always explicit. + The matrix is of CSR format. """ return self.fit(X).transform(X) @@ -370,7 +374,7 @@ class RadiusNeighborsTransformer(NeighborsBase, RadiusNeighborsMixin, required to store the tree. The optimal value depends on the nature of the problem. - metric : string or callable, default='minkowski' + metric : str or callable, default='minkowski' metric to use for distance computation. Any metric from scikit-learn or scipy.spatial.distance can be used. @@ -437,10 +441,11 @@ def transform(self, X): Returns ------- - Xt : CSR sparse graph of shape (n_samples_transform, n_samples_fit) + Xt : sparse matrix of shape (n_samples_transform, n_samples_fit) Xt[i, j] is assigned the weight of edge that connects i to j. Only the neighbors have an explicit value. The diagonal is always explicit. + The matrix is of CSR format. """ check_is_fitted(self) return self.radius_neighbors_graph(X, mode=self.mode, @@ -461,9 +466,10 @@ def fit_transform(self, X, y=None): Returns ------- - Xt : CSR sparse graph, shape (n_samples, n_samples) + Xt : sparse matrix of shape (n_samples, n_samples) Xt[i, j] is assigned the weight of edge that connects i to j. Only the neighbors have an explicit value. The diagonal is always explicit. + The matrix is of CSR format. """ return self.fit(X).transform(X) diff --git a/sklearn/neighbors/_kde.py b/sklearn/neighbors/_kde.py index 6b1e2660c2014..3404a9768f36a 100644 --- a/sklearn/neighbors/_kde.py +++ b/sklearn/neighbors/_kde.py @@ -152,7 +152,7 @@ def fit(self, X, y=None, sample_weight=None): Returns instance of object. """ algorithm = self._choose_algorithm(self.algorithm, self.metric) - X = check_array(X, order='C', dtype=DTYPE) + X = self._validate_data(X, order='C', dtype=DTYPE) if sample_weight is not None: sample_weight = _check_sample_weight(sample_weight, X, DTYPE) @@ -230,11 +230,11 @@ def sample(self, n_samples=1, random_state=None): n_samples : int, optional Number of samples to generate. Defaults to 1. - random_state : int, RandomState instance or None. default to None - If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random - number generator; If None, the random number generator is the - RandomState instance used by `np.random`. + random_state : int, RandomState instance, default=None + Determines random number generation used to generate + random samples. Pass an int for reproducible results + across multiple function calls. + See :term: `Glossary `. Returns ------- diff --git a/sklearn/neighbors/_lof.py b/sklearn/neighbors/_lof.py index e77d65711cc43..fc27b7ed69420 100644 --- a/sklearn/neighbors/_lof.py +++ b/sklearn/neighbors/_lof.py @@ -35,12 +35,12 @@ class LocalOutlierFactor(NeighborsBase, KNeighborsMixin, UnsupervisedMixin, Parameters ---------- - n_neighbors : int, optional (default=20) + n_neighbors : int, default=20 Number of neighbors to use by default for :meth:`kneighbors` queries. If n_neighbors is larger than the number of samples provided, all samples will be used. - algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional + algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto' Algorithm used to compute the nearest neighbors: - 'ball_tree' will use :class:`BallTree` @@ -52,13 +52,13 @@ class LocalOutlierFactor(NeighborsBase, KNeighborsMixin, UnsupervisedMixin, Note: fitting on sparse input will override the setting of this parameter, using brute force. - leaf_size : int, optional (default=30) + leaf_size : int, default=30 Leaf size passed to :class:`BallTree` or :class:`KDTree`. This can affect the speed of the construction and query, as well as the memory required to store the tree. The optimal value depends on the nature of the problem. - metric : string or callable, default 'minkowski' + metric : str or callable, default='minkowski' metric used for the distance computation. Any metric from scikit-learn or scipy.spatial.distance can be used. @@ -87,16 +87,16 @@ class LocalOutlierFactor(NeighborsBase, KNeighborsMixin, UnsupervisedMixin, metrics: https://docs.scipy.org/doc/scipy/reference/spatial.distance.html - p : integer, optional (default=2) + p : int, default=2 Parameter for the Minkowski metric from :func:`sklearn.metrics.pairwise.pairwise_distances`. When p = 1, this is equivalent to using manhattan_distance (l1), and euclidean_distance (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used. - metric_params : dict, optional (default=None) + metric_params : dict, default=None Additional keyword arguments for the metric function. - contamination : 'auto' or float, optional (default='auto') + contamination : 'auto' or float, default='auto' The amount of contamination of the data set, i.e. the proportion of outliers in the data set. When fitting this is used to define the threshold on the scores of the samples. @@ -109,14 +109,14 @@ class LocalOutlierFactor(NeighborsBase, KNeighborsMixin, UnsupervisedMixin, The default value of ``contamination`` changed from 0.1 to ``'auto'``. - novelty : boolean, default False + novelty : bool, default=False By default, LocalOutlierFactor is only meant to be used for outlier detection (novelty=False). Set novelty to True if you want to use LocalOutlierFactor for novelty detection. In this case be aware that that you should only use predict, decision_function and score_samples on new unseen data and not on the training set. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of parallel jobs to run for neighbors search. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` @@ -124,17 +124,18 @@ class LocalOutlierFactor(NeighborsBase, KNeighborsMixin, UnsupervisedMixin, Attributes ---------- - negative_outlier_factor_ : numpy array, shape (n_samples,) + negative_outlier_factor_ : ndarray of shape (n_samples,) The opposite LOF of the training samples. The higher, the more normal. - Inliers tend to have a LOF score close to 1 (``negative_outlier_factor_`` - close to -1), while outliers tend to have a larger LOF score. + Inliers tend to have a LOF score close to 1 + (``negative_outlier_factor_`` close to -1), while outliers tend to have + a larger LOF score. The local outlier factor (LOF) of a sample captures its supposed 'degree of abnormality'. It is the average of the ratio of the local reachability density of a sample and those of its k-nearest neighbors. - n_neighbors_ : integer + n_neighbors_ : int The actual number of neighbors used for :meth:`kneighbors` queries. offset_ : float @@ -182,16 +183,16 @@ def fit_predict(self): Parameters ---------- - X : array-like, shape (n_samples, n_features), default=None + X : array-like of shape (n_samples, n_features), default=None The query sample or samples to compute the Local Outlier Factor w.r.t. to the training samples. y : Ignored - not used, present for API consistency by convention. + Not used, present for API consistency by convention. Returns ------- - is_inlier : array, shape (n_samples,) + is_inlier : ndarray of shape (n_samples,) Returns -1 for anomalies/outliers and 1 for inliers. """ @@ -213,13 +214,13 @@ def _fit_predict(self, X, y=None): Parameters ---------- - X : array-like, shape (n_samples, n_features), default=None + X : array-like of shape (n_samples, n_features), default=None The query sample or samples to compute the Local Outlier Factor w.r.t. to the training samples. Returns ------- - is_inlier : array, shape (n_samples,) + is_inlier : ndarray of shape (n_samples,) Returns -1 for anomalies/outliers and 1 for inliers. """ @@ -233,12 +234,13 @@ def fit(self, X, y=None): Parameters ---------- - X : {array-like, sparse matrix, BallTree, KDTree} - Training data. If array or matrix, shape [n_samples, n_features], - or [n_samples, n_samples] if metric='precomputed'. + X : BallTree, KDTree or {array-like, sparse matrix} of shape \ + (n_samples, n_features) or (n_samples, n_samples) + Training data. If array or matrix, the shape is (n_samples, + n_features), or (n_samples, n_samples) if metric='precomputed'. y : Ignored - not used, present for API consistency by convention. + Not used, present for API consistency by convention. Returns ------- @@ -290,13 +292,13 @@ def predict(self): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) The query sample or samples to compute the Local Outlier Factor w.r.t. to the training samples. Returns ------- - is_inlier : array, shape (n_samples,) + is_inlier : ndarray of shape (n_samples,) Returns -1 for anomalies/outliers and +1 for inliers. """ if not self.novelty: @@ -315,14 +317,14 @@ def _predict(self, X=None): Parameters ---------- - X : array-like, shape (n_samples, n_features), default=None + X : array-like of shape (n_samples, n_features), default=None The query sample or samples to compute the Local Outlier Factor w.r.t. to the training samples. If None, makes prediction on the training data without considering them as their own neighbors. Returns ------- - is_inlier : array, shape (n_samples,) + is_inlier : ndarray of shape (n_samples,) Returns -1 for anomalies/outliers and +1 for inliers. """ check_is_fitted(self) @@ -352,13 +354,13 @@ def decision_function(self): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) The query sample or samples to compute the Local Outlier Factor w.r.t. the training samples. Returns ------- - shifted_opposite_lof_scores : array, shape (n_samples,) + shifted_opposite_lof_scores : ndarray of shape (n_samples,) The shifted opposite of the Local Outlier Factor of each input samples. The lower, the more abnormal. Negative scores represent outliers, positive scores represent inliers. @@ -388,13 +390,13 @@ def _decision_function(self, X): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) The query sample or samples to compute the Local Outlier Factor w.r.t. the training samples. Returns ------- - shifted_opposite_lof_scores : array, shape (n_samples,) + shifted_opposite_lof_scores : ndarray of shape (n_samples,) The shifted opposite of the Local Outlier Factor of each input samples. The lower, the more abnormal. Negative scores represent outliers, positive scores represent inliers. @@ -419,13 +421,13 @@ def score_samples(self): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) The query sample or samples to compute the Local Outlier Factor w.r.t. the training samples. Returns ------- - opposite_lof_scores : array, shape (n_samples,) + opposite_lof_scores : ndarray of shape (n_samples,) The opposite of the Local Outlier Factor of each input samples. The lower, the more abnormal. """ @@ -455,13 +457,13 @@ def _score_samples(self, X): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) The query sample or samples to compute the Local Outlier Factor w.r.t. the training samples. Returns ------- - opposite_lof_scores : array, shape (n_samples,) + opposite_lof_scores : ndarray of shape (n_samples,) The opposite of the Local Outlier Factor of each input samples. The lower, the more abnormal. """ @@ -487,17 +489,17 @@ def _local_reachability_density(self, distances_X, neighbors_indices): Parameters ---------- - distances_X : array, shape (n_queries, self.n_neighbors) + distances_X : ndarray of shape (n_queries, self.n_neighbors) Distances to the neighbors (in the training samples `self._fit_X`) of each query point to compute the LRD. - neighbors_indices : array, shape (n_queries, self.n_neighbors) + neighbors_indices : ndarray of shape (n_queries, self.n_neighbors) Neighbors indices (of each query point) among training samples self._fit_X. Returns ------- - local_reachability_density : array, shape (n_queries,) + local_reachability_density : ndarray of shape (n_queries,) The local reachability density of each sample. """ dist_k = self._distances_fit_X_[neighbors_indices, diff --git a/sklearn/neighbors/_nca.py b/sklearn/neighbors/_nca.py index ea90a43b3b36f..b9d2de01c958d 100644 --- a/sklearn/neighbors/_nca.py +++ b/sklearn/neighbors/_nca.py @@ -39,11 +39,12 @@ class NeighborhoodComponentsAnalysis(TransformerMixin, BaseEstimator): Parameters ---------- - n_components : int, optional (default=None) + n_components : int, default=None Preferred dimensionality of the projected space. If None it will be set to ``n_features``. - init : string or numpy array, optional (default='auto') + init : {'auto', 'pca', 'lda', 'identity', 'random'} or ndarray of shape \ + (n_features_a, n_features_b), default='auto' Initialization of the linear transformation. Possible options are 'auto', 'pca', 'lda', 'identity', 'random', and a numpy array of shape (n_features_a, n_features_b). @@ -83,40 +84,42 @@ class NeighborhoodComponentsAnalysis(TransformerMixin, BaseEstimator): :meth:`fit` and n_features_a must be less than or equal to that. If ``n_components`` is not None, n_features_a must match it. - warm_start : bool, optional, (default=False) + warm_start : bool, default=False If True and :meth:`fit` has been called before, the solution of the previous call to :meth:`fit` is used as the initial linear transformation (``n_components`` and ``init`` will be ignored). - max_iter : int, optional (default=50) + max_iter : int, default=50 Maximum number of iterations in the optimization. - tol : float, optional (default=1e-5) + tol : float, default=1e-5 Convergence tolerance for the optimization. - callback : callable, optional (default=None) + callback : callable, default=None If not None, this function is called after every iteration of the optimizer, taking as arguments the current solution (flattened transformation matrix) and the number of iterations. This might be useful in case one wants to examine or store the transformation found after each iteration. - verbose : int, optional (default=0) + verbose : int, default=0 If 0, no progress messages will be printed. If 1, progress messages will be printed to stdout. If > 1, progress messages will be printed and the ``disp`` parameter of :func:`scipy.optimize.minimize` will be set to ``verbose - 2``. - random_state : int or numpy.RandomState or None, optional (default=None) + random_state : int or numpy.RandomState, default=None A pseudo random number generator object or a seed for it if int. If ``init='random'``, ``random_state`` is used to initialize the random transformation. If ``init='pca'``, ``random_state`` is passed as an - argument to PCA when initializing the transformation. + argument to PCA when initializing the transformation. Pass an int + for reproducible results across multiple function calls. + See :term: `Glossary `. Attributes ---------- - components_ : array, shape (n_components, n_features) + components_ : ndarray of shape (n_components, n_features) The linear transformation learned during fitting. n_iter_ : int @@ -176,10 +179,10 @@ def fit(self, X, y): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) The training samples. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) The corresponding training labels. Returns @@ -244,12 +247,12 @@ def transform(self, X): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Data samples. Returns ------- - X_embedded: array, shape (n_samples, n_components) + X_embedded: ndarray of shape (n_samples, n_components) The data samples transformed. Raises @@ -268,22 +271,22 @@ def _validate_params(self, X, y): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) The training samples. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) The corresponding training labels. Returns ------- - X : array, shape (n_samples, n_features) + X : ndarray of shape (n_samples, n_features) The validated training samples. - y : array, shape (n_samples,) + y : ndarray of shape (n_samples,) The validated training labels, encoded to be integers in the range(0, n_classes). - init : string or numpy array of shape (n_features_a, n_features_b) + init : str or ndarray of shape (n_features_a, n_features_b) The validated initialization of the linear transformation. Raises @@ -297,7 +300,7 @@ def _validate_params(self, X, y): """ # Validate the inputs X and y, and converts y to numerical classes. - X, y = check_X_y(X, y, ensure_min_samples=2) + X, y = self._validate_data(X, y, ensure_min_samples=2) check_classification_targets(y) y = LabelEncoder().fit_transform(y) @@ -377,18 +380,18 @@ def _initialize(self, X, y, init): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) The training samples. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) The training labels. - init : string or numpy array of shape (n_features_a, n_features_b) + init : str or ndarray of shape (n_features_a, n_features_b) The validated initialization of the linear transformation. Returns ------- - transformation : array, shape (n_components, n_features) + transformation : ndarray of shape (n_components, n_features) The initialized linear transformation. """ @@ -443,7 +446,7 @@ def _callback(self, transformation): Parameters ---------- - transformation : array, shape=(n_components * n_features,) + transformation : ndarray of shape (n_components * n_features,) The solution computed by the optimizer in this iteration. """ if self.callback is not None: @@ -456,14 +459,14 @@ def _loss_grad_lbfgs(self, transformation, X, same_class_mask, sign=1.0): Parameters ---------- - transformation : array, shape (n_components * n_features,) + transformation : ndarray of shape (n_components * n_features,) The raveled linear transformation on which to compute loss and evaluate gradient. - X : array, shape (n_samples, n_features) + X : ndarray of shape (n_samples, n_features) The training samples. - same_class_mask : array, shape (n_samples, n_samples) + same_class_mask : ndarray of shape (n_samples, n_samples) A mask where ``mask[i, j] == 1`` if ``X[i]`` and ``X[j]`` belong to the same class, and ``0`` otherwise. @@ -472,7 +475,7 @@ def _loss_grad_lbfgs(self, transformation, X, same_class_mask, sign=1.0): loss : float The loss computed for the given transformation. - gradient : array, shape (n_components * n_features,) + gradient : ndarray of shape (n_components * n_features,) The new (flattened) gradient of the loss. """ diff --git a/sklearn/neighbors/_nearest_centroid.py b/sklearn/neighbors/_nearest_centroid.py index ce26db87b370a..48712c1fcfb44 100644 --- a/sklearn/neighbors/_nearest_centroid.py +++ b/sklearn/neighbors/_nearest_centroid.py @@ -30,7 +30,7 @@ class NearestCentroid(ClassifierMixin, BaseEstimator): Parameters ---------- - metric : string, or callable + metric : str or callable The metric to use when calculating distance between instances in a feature array. If metric is a string or callable, it must be one of the options allowed by metrics.pairwise.pairwise_distances for its @@ -41,7 +41,7 @@ class NearestCentroid(ClassifierMixin, BaseEstimator): If the "manhattan" metric is provided, this centroid is the median and for all other metrics, the centroid is now set to be the mean. - shrink_threshold : float, optional (default = None) + shrink_threshold : float, default=None Threshold for shrinking centroids to remove features. Attributes @@ -96,7 +96,7 @@ def fit(self, X, y): Training vector, where n_samples is the number of samples and n_features is the number of features. Note that centroid shrinking cannot be used with sparse matrices. - y : array, shape = [n_samples] + y : array-like of shape (n_samples,) Target values (integers) """ if self.metric == 'precomputed': @@ -104,9 +104,9 @@ def fit(self, X, y): # If X is sparse and the metric is "manhattan", store it in a csc # format is easier to calculate the median. if self.metric == 'manhattan': - X, y = check_X_y(X, y, ['csc']) + X, y = self._validate_data(X, y, accept_sparse=['csc']) else: - X, y = check_X_y(X, y, ['csr', 'csc']) + X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc']) is_X_sparse = sp.issparse(X) if is_X_sparse and self.shrink_threshold: raise ValueError("threshold shrinking not supported" diff --git a/sklearn/neighbors/_quad_tree.pyx b/sklearn/neighbors/_quad_tree.pyx index af510c317d639..5623799124f7c 100644 --- a/sklearn/neighbors/_quad_tree.pyx +++ b/sklearn/neighbors/_quad_tree.pyx @@ -97,7 +97,7 @@ cdef class _QuadTree: return self._get_cell_ndarray()['is_leaf'][:self.cell_count] def build_tree(self, X): - """Build a tree from an arary of points X.""" + """Build a tree from an array of points X.""" cdef: int i DTYPE_t[3] pt diff --git a/sklearn/neighbors/_regression.py b/sklearn/neighbors/_regression.py index caaf3da7d74fe..00d8f10c8880d 100644 --- a/sklearn/neighbors/_regression.py +++ b/sklearn/neighbors/_regression.py @@ -30,12 +30,14 @@ class KNeighborsRegressor(NeighborsBase, KNeighborsMixin, Read more in the :ref:`User Guide `. + .. versionadded:: 0.9 + Parameters ---------- - n_neighbors : int, optional (default = 5) + n_neighbors : int, default=5 Number of neighbors to use by default for :meth:`kneighbors` queries. - weights : str or callable + weights : {'uniform', 'distance'} or callable, default='uniform' weight function used in prediction. Possible values: - 'uniform' : uniform weights. All points in each neighborhood @@ -49,7 +51,7 @@ class KNeighborsRegressor(NeighborsBase, KNeighborsMixin, Uniform weights are used by default. - algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional + algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto' Algorithm used to compute the nearest neighbors: - 'ball_tree' will use :class:`BallTree` @@ -61,18 +63,18 @@ class KNeighborsRegressor(NeighborsBase, KNeighborsMixin, Note: fitting on sparse input will override the setting of this parameter, using brute force. - leaf_size : int, optional (default = 30) + leaf_size : int, default=30 Leaf size passed to BallTree or KDTree. This can affect the speed of the construction and query, as well as the memory required to store the tree. The optimal value depends on the nature of the problem. - p : integer, optional (default = 2) + p : int, default=2 Power parameter for the Minkowski metric. When p = 1, this is equivalent to using manhattan_distance (l1), and euclidean_distance (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used. - metric : string or callable, default 'minkowski' + metric : str or callable, default='minkowski' the distance metric to use for the tree. The default metric is minkowski, and with p=2 is equivalent to the standard Euclidean metric. See the documentation of the DistanceMetric class for a @@ -81,10 +83,10 @@ class KNeighborsRegressor(NeighborsBase, KNeighborsMixin, must be square during fit. X may be a :term:`Glossary `, in which case only "nonzero" elements may be considered neighbors. - metric_params : dict, optional (default = None) + metric_params : dict, default=None Additional keyword arguments for the metric function. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of parallel jobs to run for neighbors search. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` @@ -93,7 +95,7 @@ class KNeighborsRegressor(NeighborsBase, KNeighborsMixin, Attributes ---------- - effective_metric_ : string or callable + effective_metric_ : str or callable The distance metric to use. It will be same as the `metric` parameter or a synonym of it, e.g. 'euclidean' if the `metric` parameter set to 'minkowski' and `p` parameter set to 2. @@ -158,14 +160,14 @@ def predict(self, X): Parameters ---------- - X : array-like, shape (n_queries, n_features), \ + X : array-like of shape (n_queries, n_features), \ or (n_queries, n_indexed) if metric == 'precomputed' Test samples. Returns ------- - y : array of int, shape = [n_queries] or [n_queries, n_outputs] - Target values + y : ndarray of shape (n_queries,) or (n_queries, n_outputs), dtype=int + Target values. """ X = check_array(X, accept_sparse='csr') @@ -203,13 +205,15 @@ class RadiusNeighborsRegressor(NeighborsBase, RadiusNeighborsMixin, Read more in the :ref:`User Guide `. + .. versionadded:: 0.9 + Parameters ---------- - radius : float, optional (default = 1.0) + radius : float, default=1.0 Range of parameter space to use by default for :meth:`radius_neighbors` queries. - weights : str or callable + weights : {'uniform', 'distance'} or callable, default='uniform' weight function used in prediction. Possible values: - 'uniform' : uniform weights. All points in each neighborhood @@ -223,7 +227,7 @@ class RadiusNeighborsRegressor(NeighborsBase, RadiusNeighborsMixin, Uniform weights are used by default. - algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional + algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto' Algorithm used to compute the nearest neighbors: - 'ball_tree' will use :class:`BallTree` @@ -235,18 +239,18 @@ class RadiusNeighborsRegressor(NeighborsBase, RadiusNeighborsMixin, Note: fitting on sparse input will override the setting of this parameter, using brute force. - leaf_size : int, optional (default = 30) + leaf_size : int, default=30 Leaf size passed to BallTree or KDTree. This can affect the speed of the construction and query, as well as the memory required to store the tree. The optimal value depends on the nature of the problem. - p : integer, optional (default = 2) + p : int, default=2 Power parameter for the Minkowski metric. When p = 1, this is equivalent to using manhattan_distance (l1), and euclidean_distance (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used. - metric : string or callable, default 'minkowski' + metric : str or callable, default='minkowski' the distance metric to use for the tree. The default metric is minkowski, and with p=2 is equivalent to the standard Euclidean metric. See the documentation of the DistanceMetric class for a @@ -255,10 +259,10 @@ class RadiusNeighborsRegressor(NeighborsBase, RadiusNeighborsMixin, must be square during fit. X may be a :term:`Glossary `, in which case only "nonzero" elements may be considered neighbors. - metric_params : dict, optional (default = None) + metric_params : dict, default=None Additional keyword arguments for the metric function. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of parallel jobs to run for neighbors search. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` @@ -266,7 +270,7 @@ class RadiusNeighborsRegressor(NeighborsBase, RadiusNeighborsMixin, Attributes ---------- - effective_metric_ : string or callable + effective_metric_ : str or callable The distance metric to use. It will be same as the `metric` parameter or a synonym of it, e.g. 'euclidean' if the `metric` parameter set to 'minkowski' and `p` parameter set to 2. @@ -320,14 +324,15 @@ def predict(self, X): Parameters ---------- - X : array-like, shape (n_queries, n_features), \ + X : array-like of shape (n_queries, n_features), \ or (n_queries, n_indexed) if metric == 'precomputed' Test samples. Returns ------- - y : array of float, shape = [n_queries] or [n_queries, n_outputs] - Target values + y : ndarray of shape (n_queries,) or (n_queries, n_outputs), \ + dtype=double + Target values. """ X = check_array(X, accept_sparse='csr') diff --git a/sklearn/neighbors/_unsupervised.py b/sklearn/neighbors/_unsupervised.py index d760840c00ea4..20be4f636c2a4 100644 --- a/sklearn/neighbors/_unsupervised.py +++ b/sklearn/neighbors/_unsupervised.py @@ -11,16 +11,18 @@ class NearestNeighbors(NeighborsBase, KNeighborsMixin, Read more in the :ref:`User Guide `. + .. versionadded:: 0.9 + Parameters ---------- - n_neighbors : int, optional (default = 5) + n_neighbors : int, default=5 Number of neighbors to use by default for :meth:`kneighbors` queries. - radius : float, optional (default = 1.0) + radius : float, default=1.0 Range of parameter space to use by default for :meth:`radius_neighbors` queries. - algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional + algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto' Algorithm used to compute the nearest neighbors: - 'ball_tree' will use :class:`BallTree` @@ -32,13 +34,13 @@ class NearestNeighbors(NeighborsBase, KNeighborsMixin, Note: fitting on sparse input will override the setting of this parameter, using brute force. - leaf_size : int, optional (default = 30) + leaf_size : int, default=30 Leaf size passed to BallTree or KDTree. This can affect the speed of the construction and query, as well as the memory required to store the tree. The optimal value depends on the nature of the problem. - metric : string or callable, default 'minkowski' + metric : str or callable, default='minkowski' the distance metric to use for the tree. The default metric is minkowski, and with p=2 is equivalent to the standard Euclidean metric. See the documentation of the DistanceMetric class for a @@ -47,16 +49,16 @@ class NearestNeighbors(NeighborsBase, KNeighborsMixin, must be square during fit. X may be a :term:`Glossary `, in which case only "nonzero" elements may be considered neighbors. - p : integer, optional (default = 2) + p : int, default=2 Parameter for the Minkowski metric from sklearn.metrics.pairwise.pairwise_distances. When p = 1, this is equivalent to using manhattan_distance (l1), and euclidean_distance (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used. - metric_params : dict, optional (default = None) + metric_params : dict, default=None Additional keyword arguments for the metric function. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of parallel jobs to run for neighbors search. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` @@ -64,7 +66,7 @@ class NearestNeighbors(NeighborsBase, KNeighborsMixin, Attributes ---------- - effective_metric_ : string + effective_metric_ : str Metric used to compute distances to neighbors. effective_metric_params_ : dict diff --git a/sklearn/neighbors/tests/test_ball_tree.py b/sklearn/neighbors/tests/test_ball_tree.py index a85fac64a12dd..8da703dbe207d 100644 --- a/sklearn/neighbors/tests/test_ball_tree.py +++ b/sklearn/neighbors/tests/test_ball_tree.py @@ -3,12 +3,9 @@ import numpy as np import pytest from numpy.testing import assert_array_almost_equal -from sklearn.neighbors._ball_tree import (BallTree, NeighborsHeap, - simultaneous_sort, kernel_norm, - nodeheap_sort, DTYPE, ITYPE) +from sklearn.neighbors._ball_tree import BallTree from sklearn.neighbors import DistanceMetric from sklearn.utils import check_random_state -from sklearn.utils._testing import assert_allclose rng = np.random.RandomState(10) V_mahalanobis = rng.rand(3, 3) @@ -33,10 +30,6 @@ 'sokalsneath'] -def dist_func(x1, x2, p): - return np.sum((x1 - x2) ** p) ** (1. / p) - - def brute_force_neighbors(X, Y, k, metric, **kwargs): D = DistanceMetric.get_metric(metric, **kwargs).pairwise(Y, X) ind = np.argsort(D, axis=1)[:, :k] @@ -63,173 +56,6 @@ def test_ball_tree_query_metrics(metric): assert_array_almost_equal(dist1, dist2) -def test_ball_tree_query_radius(n_samples=100, n_features=10): - rng = check_random_state(0) - X = 2 * rng.random_sample(size=(n_samples, n_features)) - 1 - query_pt = np.zeros(n_features, dtype=float) - - eps = 1E-15 # roundoff error can cause test to fail - bt = BallTree(X, leaf_size=5) - rad = np.sqrt(((X - query_pt) ** 2).sum(1)) - - for r in np.linspace(rad[0], rad[-1], 100): - ind = bt.query_radius([query_pt], r + eps)[0] - i = np.where(rad <= r + eps)[0] - - ind.sort() - i.sort() - - assert_array_almost_equal(i, ind) - - -def test_ball_tree_query_radius_distance(n_samples=100, n_features=10): - rng = check_random_state(0) - X = 2 * rng.random_sample(size=(n_samples, n_features)) - 1 - query_pt = np.zeros(n_features, dtype=float) - - eps = 1E-15 # roundoff error can cause test to fail - bt = BallTree(X, leaf_size=5) - rad = np.sqrt(((X - query_pt) ** 2).sum(1)) - - for r in np.linspace(rad[0], rad[-1], 100): - ind, dist = bt.query_radius([query_pt], r + eps, return_distance=True) - - ind = ind[0] - dist = dist[0] - - d = np.sqrt(((query_pt - X[ind]) ** 2).sum(1)) - - assert_array_almost_equal(d, dist) - - -def compute_kernel_slow(Y, X, kernel, h): - d = np.sqrt(((Y[:, None, :] - X) ** 2).sum(-1)) - norm = kernel_norm(h, X.shape[1], kernel) - - if kernel == 'gaussian': - return norm * np.exp(-0.5 * (d * d) / (h * h)).sum(-1) - elif kernel == 'tophat': - return norm * (d < h).sum(-1) - elif kernel == 'epanechnikov': - return norm * ((1.0 - (d * d) / (h * h)) * (d < h)).sum(-1) - elif kernel == 'exponential': - return norm * (np.exp(-d / h)).sum(-1) - elif kernel == 'linear': - return norm * ((1 - d / h) * (d < h)).sum(-1) - elif kernel == 'cosine': - return norm * (np.cos(0.5 * np.pi * d / h) * (d < h)).sum(-1) - else: - raise ValueError('kernel not recognized') - - -@pytest.mark.parametrize("kernel", ['gaussian', 'tophat', 'epanechnikov', - 'exponential', 'linear', 'cosine']) -@pytest.mark.parametrize("h", [0.01, 0.1, 1]) -@pytest.mark.parametrize("rtol", [0, 1E-5]) -@pytest.mark.parametrize("atol", [1E-6, 1E-2]) -@pytest.mark.parametrize("breadth_first", [True, False]) -def test_ball_tree_kde(kernel, h, rtol, atol, breadth_first, n_samples=100, - n_features=3): - rng = np.random.RandomState(0) - X = rng.random_sample((n_samples, n_features)) - Y = rng.random_sample((n_samples, n_features)) - bt = BallTree(X, leaf_size=10) - - dens_true = compute_kernel_slow(Y, X, kernel, h) - - dens = bt.kernel_density(Y, h, atol=atol, rtol=rtol, - kernel=kernel, - breadth_first=breadth_first) - assert_allclose(dens, dens_true, - atol=atol, rtol=max(rtol, 1e-7)) - - -def test_gaussian_kde(n_samples=1000): - # Compare gaussian KDE results to scipy.stats.gaussian_kde - from scipy.stats import gaussian_kde - rng = check_random_state(0) - x_in = rng.normal(0, 1, n_samples) - x_out = np.linspace(-5, 5, 30) - - for h in [0.01, 0.1, 1]: - bt = BallTree(x_in[:, None]) - gkde = gaussian_kde(x_in, bw_method=h / np.std(x_in)) - - dens_bt = bt.kernel_density(x_out[:, None], h) / n_samples - dens_gkde = gkde.evaluate(x_out) - - assert_array_almost_equal(dens_bt, dens_gkde, decimal=3) - - -def test_ball_tree_two_point(n_samples=100, n_features=3): - rng = check_random_state(0) - X = rng.random_sample((n_samples, n_features)) - Y = rng.random_sample((n_samples, n_features)) - r = np.linspace(0, 1, 10) - bt = BallTree(X, leaf_size=10) - - D = DistanceMetric.get_metric("euclidean").pairwise(Y, X) - counts_true = [(D <= ri).sum() for ri in r] - - def check_two_point(r, dualtree): - counts = bt.two_point_correlation(Y, r=r, dualtree=dualtree) - assert_array_almost_equal(counts, counts_true) - - for dualtree in (True, False): - check_two_point(r, dualtree) - - - - -def test_neighbors_heap(n_pts=5, n_nbrs=10): - heap = NeighborsHeap(n_pts, n_nbrs) - - for row in range(n_pts): - d_in = rng.random_sample(2 * n_nbrs).astype(DTYPE, copy=False) - i_in = np.arange(2 * n_nbrs, dtype=ITYPE) - for d, i in zip(d_in, i_in): - heap.push(row, d, i) - - ind = np.argsort(d_in) - d_in = d_in[ind] - i_in = i_in[ind] - - d_heap, i_heap = heap.get_arrays(sort=True) - - assert_array_almost_equal(d_in[:n_nbrs], d_heap[row]) - assert_array_almost_equal(i_in[:n_nbrs], i_heap[row]) - - -def test_node_heap(n_nodes=50): - vals = rng.random_sample(n_nodes).astype(DTYPE, copy=False) - - i1 = np.argsort(vals) - vals2, i2 = nodeheap_sort(vals) - - assert_array_almost_equal(i1, i2) - assert_array_almost_equal(vals[i1], vals2) - - -def test_simultaneous_sort(n_rows=10, n_pts=201): - dist = rng.random_sample((n_rows, n_pts)).astype(DTYPE, copy=False) - ind = (np.arange(n_pts) + np.zeros((n_rows, 1))).astype(ITYPE, copy=False) - - dist2 = dist.copy() - ind2 = ind.copy() - - # simultaneous sort rows using function - simultaneous_sort(dist, ind) - - # simultaneous sort rows using numpy - i = np.argsort(dist2, axis=1) - row_ind = np.arange(n_rows)[:, None] - dist2 = dist2[row_ind, i] - ind2 = ind2[row_ind, i] - - assert_array_almost_equal(dist, dist2) - assert_array_almost_equal(ind, ind2) - - def test_query_haversine(): rng = check_random_state(0) X = 2 * np.pi * rng.random_sample((40, 2)) diff --git a/sklearn/neighbors/tests/test_dist_metrics.py b/sklearn/neighbors/tests/test_dist_metrics.py index b7939d2a0e680..7a2ec2ce4cd73 100644 --- a/sklearn/neighbors/tests/test_dist_metrics.py +++ b/sklearn/neighbors/tests/test_dist_metrics.py @@ -191,7 +191,7 @@ def wrong_distance(x, y): def test_input_data_size(): # Regression test for #6288 - # Previoulsly, a metric requiring a particular input dimension would fail + # Previously, a metric requiring a particular input dimension would fail def custom_metric(x, y): assert x.shape[0] == 3 return np.sum((x - y) ** 2) @@ -199,6 +199,6 @@ def custom_metric(x, y): rng = check_random_state(0) X = rng.rand(10, 3) - pyfunc = DistanceMetric.get_metric("pyfunc", func=dist_func, p=2) + pyfunc = DistanceMetric.get_metric("pyfunc", func=custom_metric) eucl = DistanceMetric.get_metric("euclidean") - assert_array_almost_equal(pyfunc.pairwise(X), eucl.pairwise(X)) + assert_array_almost_equal(pyfunc.pairwise(X), eucl.pairwise(X) ** 2) diff --git a/sklearn/neighbors/tests/test_kd_tree.py b/sklearn/neighbors/tests/test_kd_tree.py index ec34abf3401fd..c5d30d0b179d3 100644 --- a/sklearn/neighbors/tests/test_kd_tree.py +++ b/sklearn/neighbors/tests/test_kd_tree.py @@ -1,190 +1,6 @@ -import numpy as np -from numpy.testing import assert_array_almost_equal - -import pytest - -from sklearn.neighbors._kd_tree import (KDTree, NeighborsHeap, - simultaneous_sort, kernel_norm, - nodeheap_sort, DTYPE, ITYPE) -from sklearn.neighbors import DistanceMetric -from sklearn.utils import check_random_state -from sklearn.utils._testing import assert_allclose - DIMENSION = 3 METRICS = {'euclidean': {}, 'manhattan': {}, 'chebyshev': {}, 'minkowski': dict(p=3)} - - -def test_kd_tree_query_radius(n_samples=100, n_features=10): - rng = check_random_state(0) - X = 2 * rng.random_sample(size=(n_samples, n_features)) - 1 - query_pt = np.zeros(n_features, dtype=float) - - eps = 1E-15 # roundoff error can cause test to fail - kdt = KDTree(X, leaf_size=5) - rad = np.sqrt(((X - query_pt) ** 2).sum(1)) - - for r in np.linspace(rad[0], rad[-1], 100): - ind = kdt.query_radius([query_pt], r + eps)[0] - i = np.where(rad <= r + eps)[0] - - ind.sort() - i.sort() - - assert_array_almost_equal(i, ind) - - -def test_kd_tree_query_radius_distance(n_samples=100, n_features=10): - rng = check_random_state(0) - X = 2 * rng.random_sample(size=(n_samples, n_features)) - 1 - query_pt = np.zeros(n_features, dtype=float) - - eps = 1E-15 # roundoff error can cause test to fail - kdt = KDTree(X, leaf_size=5) - rad = np.sqrt(((X - query_pt) ** 2).sum(1)) - - for r in np.linspace(rad[0], rad[-1], 100): - ind, dist = kdt.query_radius([query_pt], r + eps, return_distance=True) - - ind = ind[0] - dist = dist[0] - - d = np.sqrt(((query_pt - X[ind]) ** 2).sum(1)) - - assert_array_almost_equal(d, dist) - - -def compute_kernel_slow(Y, X, kernel, h): - d = np.sqrt(((Y[:, None, :] - X) ** 2).sum(-1)) - norm = kernel_norm(h, X.shape[1], kernel) - - if kernel == 'gaussian': - return norm * np.exp(-0.5 * (d * d) / (h * h)).sum(-1) - elif kernel == 'tophat': - return norm * (d < h).sum(-1) - elif kernel == 'epanechnikov': - return norm * ((1.0 - (d * d) / (h * h)) * (d < h)).sum(-1) - elif kernel == 'exponential': - return norm * (np.exp(-d / h)).sum(-1) - elif kernel == 'linear': - return norm * ((1 - d / h) * (d < h)).sum(-1) - elif kernel == 'cosine': - return norm * (np.cos(0.5 * np.pi * d / h) * (d < h)).sum(-1) - else: - raise ValueError('kernel not recognized') - - -def check_results(kernel, h, atol, rtol, breadth_first, Y, kdt, dens_true): - dens = kdt.kernel_density(Y, h, atol=atol, rtol=rtol, - kernel=kernel, - breadth_first=breadth_first) - assert_allclose(dens, dens_true, atol=atol, - rtol=max(rtol, 1e-7)) - - -@pytest.mark.parametrize('kernel', - ['gaussian', 'tophat', 'epanechnikov', - 'exponential', 'linear', 'cosine']) -@pytest.mark.parametrize('h', [0.01, 0.1, 1]) -def test_kd_tree_kde(kernel, h): - n_samples, n_features = (100, 3) - rng = check_random_state(0) - X = rng.random_sample((n_samples, n_features)) - Y = rng.random_sample((n_samples, n_features)) - kdt = KDTree(X, leaf_size=10) - - dens_true = compute_kernel_slow(Y, X, kernel, h) - - for rtol in [0, 1E-5]: - for atol in [1E-6, 1E-2]: - for breadth_first in (True, False): - check_results(kernel, h, atol, rtol, - breadth_first, Y, kdt, dens_true) - - -def test_gaussian_kde(n_samples=1000): - # Compare gaussian KDE results to scipy.stats.gaussian_kde - from scipy.stats import gaussian_kde - rng = check_random_state(0) - x_in = rng.normal(0, 1, n_samples) - x_out = np.linspace(-5, 5, 30) - - for h in [0.01, 0.1, 1]: - kdt = KDTree(x_in[:, None]) - gkde = gaussian_kde(x_in, bw_method=h / np.std(x_in)) - - dens_kdt = kdt.kernel_density(x_out[:, None], h) / n_samples - dens_gkde = gkde.evaluate(x_out) - - assert_array_almost_equal(dens_kdt, dens_gkde, decimal=3) - - -@pytest.mark.parametrize('dualtree', (True, False)) -def test_kd_tree_two_point(dualtree): - n_samples, n_features = (100, 3) - rng = check_random_state(0) - X = rng.random_sample((n_samples, n_features)) - Y = rng.random_sample((n_samples, n_features)) - r = np.linspace(0, 1, 10) - kdt = KDTree(X, leaf_size=10) - - D = DistanceMetric.get_metric("euclidean").pairwise(Y, X) - counts_true = [(D <= ri).sum() for ri in r] - - counts = kdt.two_point_correlation(Y, r=r, dualtree=dualtree) - assert_array_almost_equal(counts, counts_true) - - -def test_neighbors_heap(n_pts=5, n_nbrs=10): - heap = NeighborsHeap(n_pts, n_nbrs) - rng = np.random.RandomState(42) - - for row in range(n_pts): - d_in = rng.random_sample(2 * n_nbrs).astype(DTYPE, copy=False) - i_in = np.arange(2 * n_nbrs, dtype=ITYPE) - for d, i in zip(d_in, i_in): - heap.push(row, d, i) - - ind = np.argsort(d_in) - d_in = d_in[ind] - i_in = i_in[ind] - - d_heap, i_heap = heap.get_arrays(sort=True) - - assert_array_almost_equal(d_in[:n_nbrs], d_heap[row]) - assert_array_almost_equal(i_in[:n_nbrs], i_heap[row]) - - -def test_node_heap(n_nodes=50): - rng = np.random.RandomState(42) - vals = rng.random_sample(n_nodes).astype(DTYPE, copy=False) - - i1 = np.argsort(vals) - vals2, i2 = nodeheap_sort(vals) - - assert_array_almost_equal(i1, i2) - assert_array_almost_equal(vals[i1], vals2) - - -def test_simultaneous_sort(n_rows=10, n_pts=201): - rng = np.random.RandomState(42) - dist = rng.random_sample((n_rows, n_pts)).astype(DTYPE, copy=False) - ind = (np.arange(n_pts) + np.zeros((n_rows, 1))).astype(ITYPE, copy=False) - - dist2 = dist.copy() - ind2 = ind.copy() - - # simultaneous sort rows using function - simultaneous_sort(dist, ind) - - # simultaneous sort rows using numpy - i = np.argsort(dist2, axis=1) - row_ind = np.arange(n_rows)[:, None] - dist2 = dist2[row_ind, i] - ind2 = ind2[row_ind, i] - - assert_array_almost_equal(dist, dist2) - assert_array_almost_equal(ind, ind2) diff --git a/sklearn/neighbors/tests/test_kde.py b/sklearn/neighbors/tests/test_kde.py index 1fdbc0f352853..6687cfa475ce8 100644 --- a/sklearn/neighbors/tests/test_kde.py +++ b/sklearn/neighbors/tests/test_kde.py @@ -12,6 +12,7 @@ import joblib +# XXX Duplicated in test_neighbors_tree, test_kde def compute_kernel_slow(Y, X, kernel, h): d = np.sqrt(((Y[:, None, :] - X) ** 2).sum(-1)) norm = kernel_norm(h, X.shape[1], kernel) / X.shape[0] @@ -209,10 +210,6 @@ def test_sample_weight_invalid(): kde = KernelDensity() data = np.reshape([1., 2., 3.], (-1, 1)) - sample_weight = [0.1, 0.2] - with pytest.raises(ValueError): - kde.fit(data, sample_weight=sample_weight) - sample_weight = [0.1, -0.2, 0.3] expected_err = "sample_weight must have positive values" with pytest.raises(ValueError, match=expected_err): diff --git a/sklearn/neighbors/tests/test_lof.py b/sklearn/neighbors/tests/test_lof.py index e8ddfb7090735..750fc57a8f457 100644 --- a/sklearn/neighbors/tests/test_lof.py +++ b/sklearn/neighbors/tests/test_lof.py @@ -7,6 +7,7 @@ import numpy as np from sklearn import neighbors +import pytest from numpy.testing import assert_array_equal from sklearn import metrics @@ -214,12 +215,12 @@ def test_novelty_true_common_tests(): check_estimator(neighbors.LocalOutlierFactor(novelty=True)) -def test_predicted_outlier_number(): +@pytest.mark.parametrize('expected_outliers', [30, 53]) +def test_predicted_outlier_number(expected_outliers): # the number of predicted outliers should be equal to the number of # expected outliers unless there are ties in the abnormality scores. X = iris.data n_samples = X.shape[0] - expected_outliers = 30 contamination = float(expected_outliers)/n_samples clf = neighbors.LocalOutlierFactor(contamination=contamination) diff --git a/sklearn/neighbors/tests/test_neighbors.py b/sklearn/neighbors/tests/test_neighbors.py index 03c79086dfedd..88e32669777a1 100644 --- a/sklearn/neighbors/tests/test_neighbors.py +++ b/sklearn/neighbors/tests/test_neighbors.py @@ -649,6 +649,30 @@ def test_radius_neighbors_boundary_handling(): assert_array_equal(results[0], [0, 1]) +def test_radius_neighbors_returns_array_of_objects(): + # check that we can pass precomputed distances to + # NearestNeighbors.radius_neighbors() + # non-regression test for + # https://github.com/scikit-learn/scikit-learn/issues/16036 + X = csr_matrix(np.ones((4, 4))) + X.setdiag([0, 0, 0, 0]) + + nbrs = neighbors.NearestNeighbors(radius=0.5, algorithm='auto', + leaf_size=30, + metric='precomputed').fit(X) + neigh_dist, neigh_ind = nbrs.radius_neighbors(X, return_distance=True) + + expected_dist = np.empty(X.shape[0], dtype=object) + expected_dist[:] = [np.array([0]), np.array([0]), np.array([0]), + np.array([0])] + expected_ind = np.empty(X.shape[0], dtype=object) + expected_ind[:] = [np.array([0]), np.array([1]), np.array([2]), + np.array([3])] + + assert_array_equal(neigh_dist, expected_dist) + assert_array_equal(neigh_ind, expected_ind) + + def test_RadiusNeighborsClassifier_multioutput(): # Test k-NN classifier on multioutput data rng = check_random_state(0) diff --git a/sklearn/neighbors/tests/test_neighbors_tree.py b/sklearn/neighbors/tests/test_neighbors_tree.py index 27ea7ee43e812..6609d9af2656f 100644 --- a/sklearn/neighbors/tests/test_neighbors_tree.py +++ b/sklearn/neighbors/tests/test_neighbors_tree.py @@ -7,11 +7,18 @@ import pytest from sklearn.neighbors import DistanceMetric -from sklearn.neighbors._ball_tree import BallTree -from sklearn.neighbors._kd_tree import KDTree +from sklearn.neighbors._ball_tree import ( + BallTree, kernel_norm, DTYPE, ITYPE, + NeighborsHeap as NeighborsHeapBT, + simultaneous_sort as simultaneous_sort_bt, + nodeheap_sort as nodeheap_sort_bt) +from sklearn.neighbors._kd_tree import ( + KDTree, NeighborsHeap as NeighborsHeapKDT, + simultaneous_sort as simultaneous_sort_kdt, + nodeheap_sort as nodeheap_sort_kdt) from sklearn.utils import check_random_state -from numpy.testing import assert_array_almost_equal +from numpy.testing import assert_array_almost_equal, assert_allclose rng = np.random.RandomState(42) V_mahalanobis = rng.rand(3, 3) @@ -35,6 +42,26 @@ def dist_func(x1, x2, p): return np.sum((x1 - x2) ** p) ** (1. / p) +def compute_kernel_slow(Y, X, kernel, h): + d = np.sqrt(((Y[:, None, :] - X) ** 2).sum(-1)) + norm = kernel_norm(h, X.shape[1], kernel) + + if kernel == 'gaussian': + return norm * np.exp(-0.5 * (d * d) / (h * h)).sum(-1) + elif kernel == 'tophat': + return norm * (d < h).sum(-1) + elif kernel == 'epanechnikov': + return norm * ((1.0 - (d * d) / (h * h)) * (d < h)).sum(-1) + elif kernel == 'exponential': + return norm * (np.exp(-d / h)).sum(-1) + elif kernel == 'linear': + return norm * ((1 - d / h) * (d < h)).sum(-1) + elif kernel == 'cosine': + return norm * (np.cos(0.5 * np.pi * d / h) * (d < h)).sum(-1) + else: + raise ValueError('kernel not recognized') + + def brute_force_neighbors(X, Y, k, metric, **kwargs): D = DistanceMetric.get_metric(metric, **kwargs).pairwise(Y, X) ind = np.argsort(D, axis=1)[:, :k] @@ -42,6 +69,162 @@ def brute_force_neighbors(X, Y, k, metric, **kwargs): return dist, ind +@pytest.mark.parametrize('Cls', [KDTree, BallTree]) +@pytest.mark.parametrize("kernel", ['gaussian', 'tophat', 'epanechnikov', + 'exponential', 'linear', 'cosine']) +@pytest.mark.parametrize("h", [0.01, 0.1, 1]) +@pytest.mark.parametrize("rtol", [0, 1E-5]) +@pytest.mark.parametrize("atol", [1E-6, 1E-2]) +@pytest.mark.parametrize("breadth_first", [True, False]) +def test_kernel_density(Cls, kernel, h, rtol, atol, breadth_first, + n_samples=100, n_features=3): + rng = check_random_state(1) + X = rng.random_sample((n_samples, n_features)) + Y = rng.random_sample((n_samples, n_features)) + dens_true = compute_kernel_slow(Y, X, kernel, h) + + tree = Cls(X, leaf_size=10) + dens = tree.kernel_density(Y, h, atol=atol, rtol=rtol, + kernel=kernel, + breadth_first=breadth_first) + assert_allclose(dens, dens_true, + atol=atol, rtol=max(rtol, 1e-7)) + + +@pytest.mark.parametrize('Cls', [KDTree, BallTree]) +def test_neighbor_tree_query_radius(Cls, n_samples=100, n_features=10): + rng = check_random_state(0) + X = 2 * rng.random_sample(size=(n_samples, n_features)) - 1 + query_pt = np.zeros(n_features, dtype=float) + + eps = 1E-15 # roundoff error can cause test to fail + tree = Cls(X, leaf_size=5) + rad = np.sqrt(((X - query_pt) ** 2).sum(1)) + + for r in np.linspace(rad[0], rad[-1], 100): + ind = tree.query_radius([query_pt], r + eps)[0] + i = np.where(rad <= r + eps)[0] + + ind.sort() + i.sort() + + assert_array_almost_equal(i, ind) + + +@pytest.mark.parametrize('Cls', [KDTree, BallTree]) +def test_neighbor_tree_query_radius_distance(Cls, n_samples=100, + n_features=10): + rng = check_random_state(0) + X = 2 * rng.random_sample(size=(n_samples, n_features)) - 1 + query_pt = np.zeros(n_features, dtype=float) + + eps = 1E-15 # roundoff error can cause test to fail + tree = Cls(X, leaf_size=5) + rad = np.sqrt(((X - query_pt) ** 2).sum(1)) + + for r in np.linspace(rad[0], rad[-1], 100): + ind, dist = tree.query_radius([query_pt], r + eps, + return_distance=True) + + ind = ind[0] + dist = dist[0] + + d = np.sqrt(((query_pt - X[ind]) ** 2).sum(1)) + + assert_array_almost_equal(d, dist) + + +@pytest.mark.parametrize('Cls', [KDTree, BallTree]) +@pytest.mark.parametrize('dualtree', (True, False)) +def test_neighbor_tree_two_point(Cls, dualtree, n_samples=100, n_features=3): + rng = check_random_state(0) + X = rng.random_sample((n_samples, n_features)) + Y = rng.random_sample((n_samples, n_features)) + r = np.linspace(0, 1, 10) + tree = Cls(X, leaf_size=10) + + D = DistanceMetric.get_metric("euclidean").pairwise(Y, X) + counts_true = [(D <= ri).sum() for ri in r] + + counts = tree.two_point_correlation(Y, r=r, dualtree=dualtree) + assert_array_almost_equal(counts, counts_true) + + +@pytest.mark.parametrize('NeighborsHeap', [NeighborsHeapBT, NeighborsHeapKDT]) +def test_neighbors_heap(NeighborsHeap, n_pts=5, n_nbrs=10): + heap = NeighborsHeap(n_pts, n_nbrs) + rng = check_random_state(0) + + for row in range(n_pts): + d_in = rng.random_sample(2 * n_nbrs).astype(DTYPE, copy=False) + i_in = np.arange(2 * n_nbrs, dtype=ITYPE) + for d, i in zip(d_in, i_in): + heap.push(row, d, i) + + ind = np.argsort(d_in) + d_in = d_in[ind] + i_in = i_in[ind] + + d_heap, i_heap = heap.get_arrays(sort=True) + + assert_array_almost_equal(d_in[:n_nbrs], d_heap[row]) + assert_array_almost_equal(i_in[:n_nbrs], i_heap[row]) + + +@pytest.mark.parametrize('nodeheap_sort', [nodeheap_sort_bt, + nodeheap_sort_kdt]) +def test_node_heap(nodeheap_sort, n_nodes=50): + rng = check_random_state(0) + vals = rng.random_sample(n_nodes).astype(DTYPE, copy=False) + + i1 = np.argsort(vals) + vals2, i2 = nodeheap_sort(vals) + + assert_array_almost_equal(i1, i2) + assert_array_almost_equal(vals[i1], vals2) + + +@pytest.mark.parametrize('simultaneous_sort', [simultaneous_sort_bt, + simultaneous_sort_kdt]) +def test_simultaneous_sort(simultaneous_sort, n_rows=10, n_pts=201): + rng = check_random_state(0) + dist = rng.random_sample((n_rows, n_pts)).astype(DTYPE, copy=False) + ind = (np.arange(n_pts) + np.zeros((n_rows, 1))).astype(ITYPE, copy=False) + + dist2 = dist.copy() + ind2 = ind.copy() + + # simultaneous sort rows using function + simultaneous_sort(dist, ind) + + # simultaneous sort rows using numpy + i = np.argsort(dist2, axis=1) + row_ind = np.arange(n_rows)[:, None] + dist2 = dist2[row_ind, i] + ind2 = ind2[row_ind, i] + + assert_array_almost_equal(dist, dist2) + assert_array_almost_equal(ind, ind2) + + +@pytest.mark.parametrize('Cls', [KDTree, BallTree]) +def test_gaussian_kde(Cls, n_samples=1000): + # Compare gaussian KDE results to scipy.stats.gaussian_kde + from scipy.stats import gaussian_kde + rng = check_random_state(0) + x_in = rng.normal(0, 1, n_samples) + x_out = np.linspace(-5, 5, 30) + + for h in [0.01, 0.1, 1]: + tree = Cls(x_in[:, None]) + gkde = gaussian_kde(x_in, bw_method=h / np.std(x_in)) + + dens_tree = tree.kernel_density(x_out[:, None], h) / n_samples + dens_gkde = gkde.evaluate(x_out) + + assert_array_almost_equal(dens_tree, dens_gkde, decimal=3) + + @pytest.mark.parametrize( 'Cls, metric', itertools.chain( diff --git a/sklearn/neural_network/_base.py b/sklearn/neural_network/_base.py index c29f6bbb161cb..466c082ed8745 100644 --- a/sklearn/neural_network/_base.py +++ b/sklearn/neural_network/_base.py @@ -212,6 +212,8 @@ def log_loss(y_true, y_prob): loss : float The degree to which the samples are correctly predicted. """ + eps = np.finfo(y_prob.dtype).eps + y_prob = np.clip(y_prob, eps, 1 - eps) if y_prob.shape[1] == 1: y_prob = np.append(1 - y_prob, y_prob, axis=1) @@ -232,7 +234,7 @@ def binary_log_loss(y_true, y_prob): y_true : array-like or label indicator matrix Ground truth (correct) labels. - y_prob : array-like of float, shape = (n_samples, n_classes) + y_prob : array-like of float, shape = (n_samples, 1) Predicted probabilities, as returned by a classifier's predict_proba method. @@ -241,6 +243,8 @@ def binary_log_loss(y_true, y_prob): loss : float The degree to which the samples are correctly predicted. """ + eps = np.finfo(y_prob.dtype).eps + y_prob = np.clip(y_prob, eps, 1 - eps) return -(xlogy(y_true, y_prob) + xlogy(1 - y_true, 1 - y_prob)).sum() / y_prob.shape[0] diff --git a/sklearn/neural_network/_multilayer_perceptron.py b/sklearn/neural_network/_multilayer_perceptron.py index b6367d32e57a9..6eb42bb455c3a 100644 --- a/sklearn/neural_network/_multilayer_perceptron.py +++ b/sklearn/neural_network/_multilayer_perceptron.py @@ -21,6 +21,7 @@ from ..preprocessing import LabelBinarizer from ..utils import gen_batches, check_random_state from ..utils import shuffle +from ..utils import _safe_indexing from ..utils import check_array, check_X_y, column_or_1d from ..exceptions import ConvergenceWarning from ..utils.extmath import safe_sparse_dot @@ -140,13 +141,13 @@ def _loss_grad_lbfgs(self, packed_coef_inter, X, y, activations, deltas, Parameters ---------- - packed_coef_inter : array-like + packed_coef_inter : ndarray A vector comprising the flattened coefficients and intercepts. - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input data. - y : array-like, shape (n_samples,) + y : ndarray of shape (n_samples,) The target values. activations : list, length = n_layers - 1 @@ -185,10 +186,10 @@ def _backprop(self, X, y, activations, deltas, coef_grads, Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input data. - y : array-like, shape (n_samples,) + y : ndarray of shape (n_samples,) The target values. activations : list, length = n_layers - 1 @@ -503,6 +504,7 @@ def _fit_stochastic(self, X, y, activations, deltas, coef_grads, y_val = None n_samples = X.shape[0] + sample_idx = np.arange(n_samples, dtype=int) if self.batch_size == 'auto': batch_size = min(200, n_samples) @@ -512,12 +514,24 @@ def _fit_stochastic(self, X, y, activations, deltas, coef_grads, try: for it in range(self.max_iter): if self.shuffle: - X, y = shuffle(X, y, random_state=self._random_state) + # Only shuffle the sample indices instead of X and y to + # reduce the memory footprint. These indices will be used + # to slice the X and y. + sample_idx = shuffle(sample_idx, + random_state=self._random_state) + accumulated_loss = 0.0 for batch_slice in gen_batches(n_samples, batch_size): - activations[0] = X[batch_slice] + if self.shuffle: + X_batch = _safe_indexing(X, sample_idx[batch_slice]) + y_batch = y[sample_idx[batch_slice]] + else: + X_batch = X[batch_slice] + y_batch = y[batch_slice] + + activations[0] = X_batch batch_loss, coef_grads, intercept_grads = self._backprop( - X[batch_slice], y[batch_slice], activations, deltas, + X_batch, y_batch, activations, deltas, coef_grads, intercept_grads) accumulated_loss += batch_loss * (batch_slice.stop - batch_slice.start) @@ -613,10 +627,10 @@ def fit(self, X, y): Parameters ---------- - X : array-like or sparse matrix, shape (n_samples, n_features) + X : ndarray or sparse matrix of shape (n_samples, n_features) The input data. - y : array-like, shape (n_samples,) or (n_samples, n_outputs) + y : ndarray of shape (n_samples,) or (n_samples, n_outputs) The target values (class labels in classification, real numbers in regression). @@ -632,10 +646,10 @@ def partial_fit(self): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input data. - y : array-like, shape (n_samples,) + y : ndarray of shape (n_samples,) The target values. Returns @@ -656,15 +670,15 @@ def _predict(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input data. Returns ------- - y_pred : array-like, shape (n_samples,) or (n_samples, n_outputs) + y_pred : ndarray of shape (n_samples,) or (n_samples, n_outputs) The decision function of the samples for each class in the model. """ - X = check_array(X, accept_sparse=['csr', 'csc', 'coo']) + X = check_array(X, accept_sparse=['csr', 'csc']) # Make sure self.hidden_layer_sizes is a list hidden_layer_sizes = self.hidden_layer_sizes @@ -698,11 +712,11 @@ class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron): Parameters ---------- - hidden_layer_sizes : tuple, length = n_layers - 2, default (100,) + hidden_layer_sizes : tuple, length = n_layers - 2, default=(100,) The ith element represents the number of neurons in the ith hidden layer. - activation : {'identity', 'logistic', 'tanh', 'relu'}, default 'relu' + activation : {'identity', 'logistic', 'tanh', 'relu'}, default='relu' Activation function for the hidden layer. - 'identity', no-op activation, useful to implement linear bottleneck, @@ -717,7 +731,7 @@ class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron): - 'relu', the rectified linear unit function, returns f(x) = max(0, x) - solver : {'lbfgs', 'sgd', 'adam'}, default 'adam' + solver : {'lbfgs', 'sgd', 'adam'}, default='adam' The solver for weight optimization. - 'lbfgs' is an optimizer in the family of quasi-Newton methods. @@ -733,15 +747,15 @@ class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron): For small datasets, however, 'lbfgs' can converge faster and perform better. - alpha : float, optional, default 0.0001 + alpha : float, default=0.0001 L2 penalty (regularization term) parameter. - batch_size : int, optional, default 'auto' + batch_size : int, default='auto' Size of minibatches for stochastic optimizers. If the solver is 'lbfgs', the classifier will not use minibatch. When set to "auto", `batch_size=min(200, n_samples)` - learning_rate : {'constant', 'invscaling', 'adaptive'}, default 'constant' + learning_rate : {'constant', 'invscaling', 'adaptive'}, default='constant' Learning rate schedule for weight updates. - 'constant' is a constant learning rate given by @@ -759,55 +773,56 @@ class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron): Only used when ``solver='sgd'``. - learning_rate_init : double, optional, default 0.001 + learning_rate_init : double, default=0.001 The initial learning rate used. It controls the step-size in updating the weights. Only used when solver='sgd' or 'adam'. - power_t : double, optional, default 0.5 + power_t : double, default=0.5 The exponent for inverse scaling learning rate. It is used in updating effective learning rate when the learning_rate is set to 'invscaling'. Only used when solver='sgd'. - max_iter : int, optional, default 200 + max_iter : int, default=200 Maximum number of iterations. The solver iterates until convergence (determined by 'tol') or this number of iterations. For stochastic solvers ('sgd', 'adam'), note that this determines the number of epochs (how many times each data point will be used), not the number of gradient steps. - shuffle : bool, optional, default True + shuffle : bool, default=True Whether to shuffle samples in each iteration. Only used when solver='sgd' or 'adam'. - random_state : int, RandomState instance or None, optional, default None - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance, default=None + Determines random number generation for weights and bias + initialization, train-test split if early stopping is used, and batch + sampling when solver='sgd' or 'adam'. + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary `. - tol : float, optional, default 1e-4 + tol : float, default=1e-4 Tolerance for the optimization. When the loss or score is not improving by at least ``tol`` for ``n_iter_no_change`` consecutive iterations, unless ``learning_rate`` is set to 'adaptive', convergence is considered to be reached and training stops. - verbose : bool, optional, default False + verbose : bool, default=False Whether to print progress messages to stdout. - warm_start : bool, optional, default False + warm_start : bool, default=False When set to True, reuse the solution of the previous call to fit as initialization, otherwise, just erase the previous solution. See :term:`the Glossary `. - momentum : float, default 0.9 + momentum : float, default=0.9 Momentum for gradient descent update. Should be between 0 and 1. Only used when solver='sgd'. - nesterovs_momentum : boolean, default True + nesterovs_momentum : boolean, default=True Whether to use Nesterov's momentum. Only used when solver='sgd' and momentum > 0. - early_stopping : bool, default False + early_stopping : bool, default=False Whether to use early stopping to terminate training when validation score is not improving. If set to true, it will automatically set aside 10% of training data as validation and terminate training when @@ -816,29 +831,29 @@ class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron): except in a multilabel setting. Only effective when solver='sgd' or 'adam' - validation_fraction : float, optional, default 0.1 + validation_fraction : float, default=0.1 The proportion of training data to set aside as validation set for early stopping. Must be between 0 and 1. Only used if early_stopping is True - beta_1 : float, optional, default 0.9 + beta_1 : float, default=0.9 Exponential decay rate for estimates of first moment vector in adam, should be in [0, 1). Only used when solver='adam' - beta_2 : float, optional, default 0.999 + beta_2 : float, default=0.999 Exponential decay rate for estimates of second moment vector in adam, should be in [0, 1). Only used when solver='adam' - epsilon : float, optional, default 1e-8 + epsilon : float, default=1e-8 Value for numerical stability in adam. Only used when solver='adam' - n_iter_no_change : int, optional, default 10 + n_iter_no_change : int, default=10 Maximum number of epochs to not meet ``tol`` improvement. Only effective when solver='sgd' or 'adam' .. versionadded:: 0.20 - max_fun : int, optional, default 15000 + max_fun : int, default=15000 Only used when solver='lbfgs'. Maximum number of loss function calls. The solver iterates until convergence (determined by 'tol'), number of iterations reaches max_iter, or this number of loss function calls. @@ -849,7 +864,7 @@ class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron): Attributes ---------- - classes_ : array or list of array of shape (n_classes,) + classes_ : ndarray or list of ndarray of shape (n_classes,) Class labels for each output. loss_ : float @@ -875,6 +890,23 @@ class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron): out_activation_ : string Name of the output activation function. + + Examples + -------- + >>> from sklearn.neural_network import MLPClassifier + >>> from sklearn.datasets import make_classification + >>> from sklearn.model_selection import train_test_split + >>> X, y = make_classification(n_samples=100, random_state=1) + >>> X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, + ... random_state=1) + >>> clf = MLPClassifier(random_state=1, max_iter=300).fit(X_train, y_train) + >>> clf.predict_proba(X_test[:1]) + array([[0.038..., 0.961...]]) + >>> clf.predict(X_test[:5, :]) + array([1, 0, 1, 0, 1]) + >>> clf.score(X_test, y_test) + 0.8... + Notes ----- MLPClassifier trains iteratively since at each time step @@ -928,8 +960,8 @@ def __init__(self, hidden_layer_sizes=(100,), activation="relu", n_iter_no_change=n_iter_no_change, max_fun=max_fun) def _validate_input(self, X, y, incremental): - X, y = check_X_y(X, y, accept_sparse=['csr', 'csc', 'coo'], - multi_output=True) + X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc'], + multi_output=True) if y.ndim == 2 and y.shape[1] == 1: y = column_or_1d(y, warn=True) @@ -959,12 +991,12 @@ def predict(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input data. Returns ------- - y : array-like, shape (n_samples,) or (n_samples, n_classes) + y : ndarray, shape (n_samples,) or (n_samples, n_classes) The predicted classes. """ check_is_fitted(self) @@ -980,10 +1012,10 @@ def fit(self, X, y): Parameters ---------- - X : array-like or sparse matrix, shape (n_samples, n_features) + X : ndarray or sparse matrix of shape (n_samples, n_features) The input data. - y : array-like, shape (n_samples,) or (n_samples, n_outputs) + y : ndarray, shape (n_samples,) or (n_samples, n_outputs) The target values (class labels in classification, real numbers in regression). @@ -1041,12 +1073,12 @@ def predict_log_proba(self, X): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : ndarray of shape (n_samples, n_features) The input data. Returns ------- - log_y_prob : array-like, shape (n_samples, n_classes) + log_y_prob : ndarray of shape (n_samples, n_classes) The predicted log-probability of the sample for each class in the model, where classes are ordered as they are in `self.classes_`. Equivalent to log(predict_proba(X)) @@ -1059,12 +1091,12 @@ def predict_proba(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input data. Returns ------- - y_prob : array-like, shape (n_samples, n_classes) + y_prob : ndarray of shape (n_samples, n_classes) The predicted probability of the sample for each class in the model, where classes are ordered as they are in `self.classes_`. """ @@ -1090,11 +1122,11 @@ class MLPRegressor(RegressorMixin, BaseMultilayerPerceptron): Parameters ---------- - hidden_layer_sizes : tuple, length = n_layers - 2, default (100,) + hidden_layer_sizes : tuple, length = n_layers - 2, default=(100,) The ith element represents the number of neurons in the ith hidden layer. - activation : {'identity', 'logistic', 'tanh', 'relu'}, default 'relu' + activation : {'identity', 'logistic', 'tanh', 'relu'}, default='relu' Activation function for the hidden layer. - 'identity', no-op activation, useful to implement linear bottleneck, @@ -1109,7 +1141,7 @@ class MLPRegressor(RegressorMixin, BaseMultilayerPerceptron): - 'relu', the rectified linear unit function, returns f(x) = max(0, x) - solver : {'lbfgs', 'sgd', 'adam'}, default 'adam' + solver : {'lbfgs', 'sgd', 'adam'}, default='adam' The solver for weight optimization. - 'lbfgs' is an optimizer in the family of quasi-Newton methods. @@ -1125,15 +1157,15 @@ class MLPRegressor(RegressorMixin, BaseMultilayerPerceptron): For small datasets, however, 'lbfgs' can converge faster and perform better. - alpha : float, optional, default 0.0001 + alpha : float, default=0.0001 L2 penalty (regularization term) parameter. - batch_size : int, optional, default 'auto' + batch_size : int, default='auto' Size of minibatches for stochastic optimizers. If the solver is 'lbfgs', the classifier will not use minibatch. When set to "auto", `batch_size=min(200, n_samples)` - learning_rate : {'constant', 'invscaling', 'adaptive'}, default 'constant' + learning_rate : {'constant', 'invscaling', 'adaptive'}, default='constant' Learning rate schedule for weight updates. - 'constant' is a constant learning rate given by @@ -1151,55 +1183,56 @@ class MLPRegressor(RegressorMixin, BaseMultilayerPerceptron): Only used when solver='sgd'. - learning_rate_init : double, optional, default 0.001 + learning_rate_init : double, default=0.001 The initial learning rate used. It controls the step-size in updating the weights. Only used when solver='sgd' or 'adam'. - power_t : double, optional, default 0.5 + power_t : double, default=0.5 The exponent for inverse scaling learning rate. It is used in updating effective learning rate when the learning_rate is set to 'invscaling'. Only used when solver='sgd'. - max_iter : int, optional, default 200 + max_iter : int, default=200 Maximum number of iterations. The solver iterates until convergence (determined by 'tol') or this number of iterations. For stochastic solvers ('sgd', 'adam'), note that this determines the number of epochs (how many times each data point will be used), not the number of gradient steps. - shuffle : bool, optional, default True + shuffle : bool, default=True Whether to shuffle samples in each iteration. Only used when solver='sgd' or 'adam'. - random_state : int, RandomState instance or None, optional, default None - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance, default=None + Determines random number generation for weights and bias + initialization, train-test split if early stopping is used, and batch + sampling when solver='sgd' or 'adam'. + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary `. - tol : float, optional, default 1e-4 + tol : float, default=1e-4 Tolerance for the optimization. When the loss or score is not improving by at least ``tol`` for ``n_iter_no_change`` consecutive iterations, unless ``learning_rate`` is set to 'adaptive', convergence is considered to be reached and training stops. - verbose : bool, optional, default False + verbose : bool, default=False Whether to print progress messages to stdout. - warm_start : bool, optional, default False + warm_start : bool, default=False When set to True, reuse the solution of the previous call to fit as initialization, otherwise, just erase the previous solution. See :term:`the Glossary `. - momentum : float, default 0.9 + momentum : float, default=0.9 Momentum for gradient descent update. Should be between 0 and 1. Only used when solver='sgd'. - nesterovs_momentum : boolean, default True + nesterovs_momentum : boolean, default=True Whether to use Nesterov's momentum. Only used when solver='sgd' and momentum > 0. - early_stopping : bool, default False + early_stopping : bool, default=False Whether to use early stopping to terminate training when validation score is not improving. If set to true, it will automatically set aside 10% of training data as validation and terminate training when @@ -1207,29 +1240,29 @@ class MLPRegressor(RegressorMixin, BaseMultilayerPerceptron): ``n_iter_no_change`` consecutive epochs. Only effective when solver='sgd' or 'adam' - validation_fraction : float, optional, default 0.1 + validation_fraction : float, default=0.1 The proportion of training data to set aside as validation set for early stopping. Must be between 0 and 1. Only used if early_stopping is True - beta_1 : float, optional, default 0.9 + beta_1 : float, default=0.9 Exponential decay rate for estimates of first moment vector in adam, should be in [0, 1). Only used when solver='adam' - beta_2 : float, optional, default 0.999 + beta_2 : float, default=0.999 Exponential decay rate for estimates of second moment vector in adam, should be in [0, 1). Only used when solver='adam' - epsilon : float, optional, default 1e-8 + epsilon : float, default=1e-8 Value for numerical stability in adam. Only used when solver='adam' - n_iter_no_change : int, optional, default 10 + n_iter_no_change : int, default=10 Maximum number of epochs to not meet ``tol`` improvement. Only effective when solver='sgd' or 'adam' .. versionadded:: 0.20 - max_fun : int, optional, default 15000 + max_fun : int, default=15000 Only used when solver='lbfgs'. Maximum number of function calls. The solver iterates until convergence (determined by 'tol'), number of iterations reaches max_iter, or this number of function calls. @@ -1263,6 +1296,20 @@ class MLPRegressor(RegressorMixin, BaseMultilayerPerceptron): out_activation_ : string Name of the output activation function. + Examples + -------- + >>> from sklearn.neural_network import MLPRegressor + >>> from sklearn.datasets import make_regression + >>> from sklearn.model_selection import train_test_split + >>> X, y = make_regression(n_samples=200, random_state=1) + >>> X_train, X_test, y_train, y_test = train_test_split(X, y, + ... random_state=1) + >>> regr = MLPRegressor(random_state=1, max_iter=500).fit(X_train, y_train) + >>> regr.predict(X_test[:2]) + array([-0.9..., -7.1...]) + >>> regr.score(X_test, y_test) + 0.4... + Notes ----- MLPRegressor trains iteratively since at each time step @@ -1321,12 +1368,12 @@ def predict(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input data. Returns ------- - y : array-like, shape (n_samples, n_outputs) + y : ndarray of shape (n_samples, n_outputs) The predicted values. """ check_is_fitted(self) @@ -1336,8 +1383,8 @@ def predict(self, X): return y_pred def _validate_input(self, X, y, incremental): - X, y = check_X_y(X, y, accept_sparse=['csr', 'csc', 'coo'], - multi_output=True, y_numeric=True) + X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc'], + multi_output=True, y_numeric=True) if y.ndim == 2 and y.shape[1] == 1: y = column_or_1d(y, warn=True) return X, y diff --git a/sklearn/neural_network/_rbm.py b/sklearn/neural_network/_rbm.py index efe3aeda951af..06e7cc71bad3c 100644 --- a/sklearn/neural_network/_rbm.py +++ b/sklearn/neural_network/_rbm.py @@ -38,28 +38,35 @@ class BernoulliRBM(TransformerMixin, BaseEstimator): Parameters ---------- - n_components : int, optional + n_components : int, default=256 Number of binary hidden units. - learning_rate : float, optional + learning_rate : float, default=0.1 The learning rate for weight updates. It is *highly* recommended to tune this hyper-parameter. Reasonable values are in the 10**[0., -3.] range. - batch_size : int, optional + batch_size : int, default=10 Number of examples per minibatch. - n_iter : int, optional + n_iter : int, default=10 Number of iterations/sweeps over the training dataset to perform during training. - verbose : int, optional + verbose : int, default=0 The verbosity level. The default, zero, means silent mode. - random_state : integer or RandomState, optional - A random number generator instance to define the state of the - random permutations generator. If an integer is given, it fixes the - seed. Defaults to the global numpy random number generator. + random_state : integer or RandomState, default=None + Determines random number generation for: + + - Gibbs sampling from visible and hidden layers. + + - Initializing components, sampling from layers during fit. + + - Corrupting the data when scoring samples. + + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary `. Attributes ---------- @@ -113,12 +120,12 @@ def transform(self, X): Parameters ---------- - X : {array-like, sparse matrix} shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The data to be transformed. Returns ------- - h : array, shape (n_samples, n_components) + h : ndarray of shape (n_samples, n_components) Latent representations of the data. """ check_is_fitted(self) @@ -131,12 +138,12 @@ def _mean_hiddens(self, v): Parameters ---------- - v : array-like, shape (n_samples, n_features) + v : ndarray of shape (n_samples, n_features) Values of the visible layer. Returns ------- - h : array-like, shape (n_samples, n_components) + h : ndarray of shape (n_samples, n_components) Corresponding mean field values for the hidden layer. """ p = safe_sparse_dot(v, self.components_.T) @@ -148,7 +155,7 @@ def _sample_hiddens(self, v, rng): Parameters ---------- - v : array-like, shape (n_samples, n_features) + v : ndarray of shape (n_samples, n_features) Values of the visible layer to sample from. rng : RandomState @@ -156,7 +163,7 @@ def _sample_hiddens(self, v, rng): Returns ------- - h : array-like, shape (n_samples, n_components) + h : ndarray of shape (n_samples, n_components) Values of the hidden layer. """ p = self._mean_hiddens(v) @@ -167,7 +174,7 @@ def _sample_visibles(self, h, rng): Parameters ---------- - h : array-like, shape (n_samples, n_components) + h : ndarray of shape (n_samples, n_components) Values of the hidden layer to sample from. rng : RandomState @@ -175,7 +182,7 @@ def _sample_visibles(self, h, rng): Returns ------- - v : array-like, shape (n_samples, n_features) + v : ndarray of shape (n_samples, n_features) Values of the visible layer. """ p = np.dot(h, self.components_) @@ -188,12 +195,12 @@ def _free_energy(self, v): Parameters ---------- - v : array-like, shape (n_samples, n_features) + v : ndarray of shape (n_samples, n_features) Values of the visible layer. Returns ------- - free_energy : array-like, shape (n_samples,) + free_energy : ndarray of shape (n_samples,) The value of the free energy. """ return (- safe_sparse_dot(v, self.intercept_visible_) @@ -205,12 +212,12 @@ def gibbs(self, v): Parameters ---------- - v : array-like, shape (n_samples, n_features) + v : ndarray of shape (n_samples, n_features) Values of the visible layer to start from. Returns ------- - v_new : array-like, shape (n_samples, n_features) + v_new : ndarray of shape (n_samples, n_features) Values of the visible layer after one Gibbs step. """ check_is_fitted(self) @@ -227,7 +234,7 @@ def partial_fit(self, X, y=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : ndarray of shape (n_samples, n_features) Training data. Returns @@ -263,7 +270,7 @@ def _fit(self, v_pos, rng): Parameters ---------- - v_pos : array-like, shape (n_samples, n_features) + v_pos : ndarray of shape (n_samples, n_features) The data to use for training. rng : RandomState @@ -290,12 +297,12 @@ def score_samples(self, X): Parameters ---------- - X : {array-like, sparse matrix} shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Values of the visible layer. Must be all-boolean (not checked). Returns ------- - pseudo_likelihood : array-like, shape (n_samples,) + pseudo_likelihood : ndarray of shape (n_samples,) Value of the pseudo-likelihood (proxy for likelihood). Notes @@ -328,7 +335,7 @@ def fit(self, X, y=None): Parameters ---------- - X : {array-like, sparse matrix} shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data. Returns @@ -336,7 +343,7 @@ def fit(self, X, y=None): self : BernoulliRBM The fitted model. """ - X = check_array(X, accept_sparse='csr', dtype=np.float64) + X = self._validate_data(X, accept_sparse='csr', dtype=np.float64) n_samples = X.shape[0] rng = check_random_state(self.random_state) @@ -365,3 +372,11 @@ def fit(self, X, y=None): begin = end return self + + def _more_tags(self): + return { + '_xfail_test': { + 'check_methods_subset_invariance': + 'fails for the decision_function method' + } + } diff --git a/sklearn/neural_network/_stochastic_optimizers.py b/sklearn/neural_network/_stochastic_optimizers.py index 3e49e94de8bd1..2da9c0b278e71 100644 --- a/sklearn/neural_network/_stochastic_optimizers.py +++ b/sklearn/neural_network/_stochastic_optimizers.py @@ -16,7 +16,7 @@ class BaseOptimizer: The concatenated list containing coefs_ and intercepts_ in MLP model. Used for initializing velocities and updating params - learning_rate_init : float, optional, default 0.1 + learning_rate_init : float, default=0.1 The initial learning rate used. It controls the step-size in updating the weights @@ -80,11 +80,11 @@ class SGDOptimizer(BaseOptimizer): The concatenated list containing coefs_ and intercepts_ in MLP model. Used for initializing velocities and updating params - learning_rate_init : float, optional, default 0.1 + learning_rate_init : float, default=0.1 The initial learning rate used. It controls the step-size in updating the weights - lr_schedule : {'constant', 'adaptive', 'invscaling'}, default 'constant' + lr_schedule : {'constant', 'adaptive', 'invscaling'}, default='constant' Learning rate schedule for weight updates. -'constant', is a constant learning rate given by @@ -100,12 +100,16 @@ class SGDOptimizer(BaseOptimizer): tol, or fail to increase validation score by tol if 'early_stopping' is on, the current learning rate is divided by 5. - momentum : float, optional, default 0.9 + momentum : float, default=0.9 Value of momentum used, must be larger than or equal to 0 - nesterov : bool, optional, default True + nesterov : bool, default=True Whether to use nesterov's momentum or not. Use nesterov's if True + power_t : float, default=0.5 + Power of time step 't' in inverse scaling. See `lr_schedule` for + more details. + Attributes ---------- learning_rate : float @@ -192,19 +196,19 @@ class AdamOptimizer(BaseOptimizer): The concatenated list containing coefs_ and intercepts_ in MLP model. Used for initializing velocities and updating params - learning_rate_init : float, optional, default 0.1 + learning_rate_init : float, default=0.001 The initial learning rate used. It controls the step-size in updating the weights - beta_1 : float, optional, default 0.9 + beta_1 : float, default=0.9 Exponential decay rate for estimates of first moment vector, should be in [0, 1) - beta_2 : float, optional, default 0.999 + beta_2 : float, default=0.999 Exponential decay rate for estimates of second moment vector, should be in [0, 1) - epsilon : float, optional, default 1e-8 + epsilon : float, default=1e-8 Value for numerical stability Attributes diff --git a/sklearn/neural_network/tests/test_base.py b/sklearn/neural_network/tests/test_base.py new file mode 100644 index 0000000000000..c803efe561faa --- /dev/null +++ b/sklearn/neural_network/tests/test_base.py @@ -0,0 +1,26 @@ +import pytest +import numpy as np + +from sklearn.neural_network._base import binary_log_loss +from sklearn.neural_network._base import log_loss + + +def test_binary_log_loss_1_prob_finite(): + # y_proba is equal to one should result in a finite logloss + y_true = np.array([[0, 0, 1]]).T + y_prob = np.array([[0.9, 1.0, 1.0]]).T + + loss = binary_log_loss(y_true, y_prob) + assert np.isfinite(loss) + + +@pytest.mark.parametrize("y_true, y_prob", [ + (np.array([[1, 0, 0], [0, 1, 0]]), + np.array([[0., 1., 0.], [0.9, 0.05, 0.05]])), + (np.array([[0, 0, 1]]).T, + np.array([[0.9, 1.0, 1.0]]).T), +]) +def test_log_loss_1_prob_finite(y_true, y_prob): + # y_proba is equal to 1 should result in a finite logloss + loss = log_loss(y_true, y_prob) + assert np.isfinite(loss) diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py index 53f69b79edb40..09a01ad69dbdd 100644 --- a/sklearn/neural_network/tests/test_mlp.py +++ b/sklearn/neural_network/tests/test_mlp.py @@ -345,7 +345,6 @@ def test_multilabel_classification(): mlp.fit(X, y).predict(X) -@pytest.mark.filterwarnings('ignore: The default value of multioutput') # 0.23 def test_multioutput_regression(): # Test that multi-output regression works as expected X, y = make_regression(n_samples=200, n_targets=5) diff --git a/sklearn/pipeline.py b/sklearn/pipeline.py index e4ac9007fe8e5..64d2de70df531 100644 --- a/sklearn/pipeline.py +++ b/sklearn/pipeline.py @@ -47,6 +47,8 @@ class Pipeline(_BaseComposition): Read more in the :ref:`User Guide `. + .. versionadded:: 0.5 + Parameters ---------- steps : list @@ -54,7 +56,7 @@ class Pipeline(_BaseComposition): chained, in the order in which they are chained, with the last object an estimator. - memory : None, str or object with the joblib.Memory interface, optional + memory : str or object with the joblib.Memory interface, default=None Used to cache the fitted transformers of the pipeline. By default, no caching is performed. If a string is given, it is the path to the caching directory. Enabling caching triggers a clone of @@ -70,7 +72,8 @@ class Pipeline(_BaseComposition): Attributes ---------- - named_steps : bunch object, a dictionary with attribute access + named_steps : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. Read-only attribute to access any step parameter by user given name. Keys are step names and values are steps parameters. @@ -138,7 +141,7 @@ def get_params(self, deep=True): Parameters ---------- - deep : boolean, optional + deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. @@ -305,7 +308,7 @@ def _fit(self, X, y=None, **fit_params): cloned_transformer = clone(transformer) else: cloned_transformer = clone(transformer) - # Fit or load from cache the current transfomer + # Fit or load from cache the current transformer X, fitted_transformer = fit_transform_one_cached( cloned_transformer, X, y, None, message_clsname='Pipeline', @@ -500,7 +503,7 @@ def score_samples(self, X): Returns ------- - y_score : ndarray, shape (n_samples,) + y_score : ndarray of shape (n_samples,) """ Xt = X for _, _, transformer in self._iter(with_final=False): @@ -625,6 +628,11 @@ def _pairwise(self): # check if first estimator expects pairwise input return getattr(self.steps[0][1], '_pairwise', False) + @property + def n_features_in_(self): + # delegate to first step (which will call _check_is_fitted) + return self.steps[0][1].n_features_in_ + def _name_estimators(estimators): """Generate names for estimators.""" @@ -662,7 +670,7 @@ def make_pipeline(*steps, **kwargs): ---------- *steps : list of estimators. - memory : None, str or object with the joblib.Memory interface, optional + memory : str or object with the joblib.Memory interface, default=None Used to cache the fitted transformers of the pipeline. By default, no caching is performed. If a string is given, it is the path to the caching directory. Enabling caching triggers a clone of @@ -672,7 +680,7 @@ def make_pipeline(*steps, **kwargs): inspect estimators within the pipeline. Caching the transformers is advantageous when fitting is time consuming. - verbose : boolean, default=False + verbose : bool, default=False If True, the time elapsed while fitting each step will be printed as it is completed. @@ -771,17 +779,17 @@ class FeatureUnion(TransformerMixin, _BaseComposition): .. versionchanged:: 0.22 Deprecated `None` as a transformer in favor of 'drop'. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None Number of jobs to run in parallel. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - transformer_weights : dict, optional + transformer_weights : dict, default=None Multiplicative weights for features per transformer. Keys are transformer names, values the weights. - verbose : boolean, optional(default=False) + verbose : bool, default=False If True, the time elapsed while fitting each transformer will be printed as it is completed. @@ -816,7 +824,7 @@ def get_params(self, deep=True): Parameters ---------- - deep : boolean, optional + deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. @@ -898,7 +906,7 @@ def fit(self, X, y=None, **fit_params): X : iterable or array-like, depending on transformers Input data, used to fit transformers. - y : array-like, shape (n_samples, ...), optional + y : array-like of shape (n_samples, n_outputs), default=None Targets for supervised learning. Returns @@ -922,12 +930,13 @@ def fit_transform(self, X, y=None, **fit_params): X : iterable or array-like, depending on transformers Input data to be transformed. - y : array-like, shape (n_samples, ...), optional + y : array-like of shape (n_samples, n_outputs), default=None Targets for supervised learning. Returns ------- - X_t : array-like or sparse matrix, shape (n_samples, sum_n_components) + X_t : array-like or sparse matrix of \ + shape (n_samples, sum_n_components) hstack of results of transformers. sum_n_components is the sum of n_components (output dimension) over transformers. """ @@ -973,7 +982,8 @@ def transform(self, X): Returns ------- - X_t : array-like or sparse matrix, shape (n_samples, sum_n_components) + X_t : array-like or sparse matrix of \ + shape (n_samples, sum_n_components) hstack of results of transformers. sum_n_components is the sum of n_components (output dimension) over transformers. """ @@ -995,6 +1005,11 @@ def _update_transformer_list(self, transformers): else next(transformers)) for name, old in self.transformer_list] + @property + def n_features_in_(self): + # X is passed to all transformers so we just delegate to the first one + return self.transformer_list[0][1].n_features_in_ + def make_union(*transformers, **kwargs): """ @@ -1008,13 +1023,13 @@ def make_union(*transformers, **kwargs): ---------- *transformers : list of estimators - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None Number of jobs to run in parallel. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. - verbose : boolean, optional(default=False) + verbose : bool, default=False If True, the time elapsed while fitting each transformer will be printed as it is completed. diff --git a/sklearn/preprocessing/_data.py b/sklearn/preprocessing/_data.py index 9514719de5a92..33e2bac562489 100644 --- a/sklearn/preprocessing/_data.py +++ b/sklearn/preprocessing/_data.py @@ -365,20 +365,19 @@ def partial_fit(self, X, y=None): " than maximum. Got %s." % str(feature_range)) if sparse.issparse(X): - raise TypeError("MinMaxScaler does no support sparse input. " - "You may consider to use MaxAbsScaler instead.") + raise TypeError("MinMaxScaler does not support sparse input. " + "Consider using MaxAbsScaler instead.") - X = check_array(X, - estimator=self, dtype=FLOAT_DTYPES, - force_all_finite="allow-nan") + first_pass = not hasattr(self, 'n_samples_seen_') + X = self._validate_data(X, reset=first_pass, + estimator=self, dtype=FLOAT_DTYPES, + force_all_finite="allow-nan") data_min = np.nanmin(X, axis=0) data_max = np.nanmax(X, axis=0) - # First pass - if not hasattr(self, 'n_samples_seen_'): + if first_pass: self.n_samples_seen_ = X.shape[0] - # Next steps else: data_min = np.minimum(self.data_min_, data_min) data_max = np.maximum(self.data_max_, data_max) @@ -695,9 +694,9 @@ def partial_fit(self, X, y=None): self : object Transformer instance. """ - X = check_array(X, accept_sparse=('csr', 'csc'), - estimator=self, dtype=FLOAT_DTYPES, - force_all_finite='allow-nan') + X = self._validate_data(X, accept_sparse=('csr', 'csc'), + estimator=self, dtype=FLOAT_DTYPES, + force_all_finite='allow-nan') # Even in the case of `with_mean=False`, we update the mean anyway # This is needed for the incremental computation of the var @@ -790,9 +789,10 @@ def transform(self, X, copy=None): check_is_fitted(self) copy = copy if copy is not None else self.copy - X = check_array(X, accept_sparse='csr', copy=copy, - estimator=self, dtype=FLOAT_DTYPES, - force_all_finite='allow-nan') + X = self._validate_data(X, reset=False, + accept_sparse='csr', copy=copy, + estimator=self, dtype=FLOAT_DTYPES, + force_all_finite='allow-nan') if sparse.issparse(X): if self.with_mean: @@ -965,9 +965,11 @@ def partial_fit(self, X, y=None): self : object Transformer instance. """ - X = check_array(X, accept_sparse=('csr', 'csc'), - estimator=self, dtype=FLOAT_DTYPES, - force_all_finite='allow-nan') + first_pass = not hasattr(self, 'n_samples_seen_') + X = self._validate_data(X, reset=first_pass, + accept_sparse=('csr', 'csc'), estimator=self, + dtype=FLOAT_DTYPES, + force_all_finite='allow-nan') if sparse.issparse(X): mins, maxs = min_max_axis(X, axis=0, ignore_nan=True) @@ -975,10 +977,8 @@ def partial_fit(self, X, y=None): else: max_abs = np.nanmax(np.abs(X), axis=0) - # First pass - if not hasattr(self, 'n_samples_seen_'): + if first_pass: self.n_samples_seen_ = X.shape[0] - # Next passes else: max_abs = np.maximum(self.max_abs_, max_abs) self.n_samples_seen_ += X.shape[0] @@ -1196,8 +1196,9 @@ def fit(self, X, y=None): """ # at fit, convert sparse matrices to csc for optimized computation of # the quantiles - X = check_array(X, accept_sparse='csc', estimator=self, - dtype=FLOAT_DTYPES, force_all_finite='allow-nan') + X = self._validate_data(X, accept_sparse='csc', estimator=self, + dtype=FLOAT_DTYPES, + force_all_finite='allow-nan') q_min, q_max = self.quantile_range if not 0 <= q_min <= q_max <= 100: @@ -1505,7 +1506,8 @@ def fit(self, X, y=None): ------- self : instance """ - n_samples, n_features = check_array(X, accept_sparse=True).shape + n_samples, n_features = self._validate_data( + X, accept_sparse=True).shape combinations = self._combinations(n_features, self.degree, self.interaction_only, self.include_bias) @@ -1716,7 +1718,8 @@ def normalize(X, norm='l2', axis=1, copy=True, return_norm=False): elif norm == 'l2': inplace_csr_row_normalize_l2(X) elif norm == 'max': - _, norms = min_max_axis(X, 1) + mins, maxes = min_max_axis(X, 1) + norms = np.maximum(abs(mins), maxes) norms_elementwise = norms.repeat(np.diff(X.indptr)) mask = norms_elementwise != 0 X.data[mask] /= norms_elementwise[mask] @@ -1726,7 +1729,7 @@ def normalize(X, norm='l2', axis=1, copy=True, return_norm=False): elif norm == 'l2': norms = row_norms(X) elif norm == 'max': - norms = np.max(X, axis=1) + norms = np.max(abs(X), axis=1) norms = _handle_zeros_in_scale(norms, copy=False) X /= norms[:, np.newaxis] @@ -1744,7 +1747,7 @@ class Normalizer(TransformerMixin, BaseEstimator): Each sample (i.e. each row of the data matrix) with at least one non zero component is rescaled independently of other samples so - that its norm (l1 or l2) equals one. + that its norm (l1, l2 or inf) equals one. This transformer is able to work both with dense numpy arrays and scipy.sparse matrix (use CSR format if you want to avoid the burden of @@ -1761,7 +1764,9 @@ class Normalizer(TransformerMixin, BaseEstimator): Parameters ---------- norm : 'l1', 'l2', or 'max', optional ('l2' by default) - The norm to use to normalize each non zero sample. + The norm to use to normalize each non zero sample. If norm='max' + is used, values will be rescaled by the maximum of the absolute + values. copy : boolean, optional, default True set to False to perform inplace row normalization and avoid a @@ -1811,7 +1816,7 @@ def fit(self, X, y=None): ---------- X : array-like """ - check_array(X, accept_sparse='csr') + self._validate_data(X, accept_sparse='csr') return self def transform(self, X, copy=None): @@ -1945,7 +1950,7 @@ def fit(self, X, y=None): ---------- X : array-like """ - check_array(X, accept_sparse='csr') + self._validate_data(X, accept_sparse='csr') return self def transform(self, X, copy=None): @@ -2025,7 +2030,7 @@ def fit(self, K, y=None): self : returns an instance of self. """ - K = check_array(K, dtype=FLOAT_DTYPES) + K = self._validate_data(K, dtype=FLOAT_DTYPES) if K.shape[0] != K.shape[1]: raise ValueError("Kernel matrix must be a square matrix." @@ -2176,11 +2181,11 @@ class QuantileTransformer(TransformerMixin, BaseEstimator): differ for value-identical sparse and dense matrices. random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by np.random. Note that this is used by subsampling and smoothing + Determines random number generation for subsampling and smoothing noise. + Please see ``subsample`` for more details. + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary ` copy : boolean, optional, (default=True) Set to False to perform inplace transformation and avoid a copy (if the @@ -2262,6 +2267,11 @@ def _dense_fit(self, X, random_state): col = col.take(subsample_idx, mode='clip') self.quantiles_.append(np.nanpercentile(col, references)) self.quantiles_ = np.transpose(self.quantiles_) + # Due to floating-point precision error in `np.nanpercentile`, + # make sure that quantiles are monotonically increasing. + # Upstream issue in numpy: + # https://github.com/numpy/numpy/issues/14685 + self.quantiles_ = np.maximum.accumulate(self.quantiles_) def _sparse_fit(self, X, random_state): """Compute percentiles for sparse matrices. @@ -2305,6 +2315,11 @@ def _sparse_fit(self, X, random_state): self.quantiles_.append( np.nanpercentile(column_data, references)) self.quantiles_ = np.transpose(self.quantiles_) + # due to floating-point precision error in `np.nanpercentile`, + # make sure the quantiles are monotonically increasing + # Upstream issue in numpy: + # https://github.com/numpy/numpy/issues/14685 + self.quantiles_ = np.maximum.accumulate(self.quantiles_) def fit(self, X, y=None): """Compute the quantiles used for transforming. @@ -2337,7 +2352,7 @@ def fit(self, X, y=None): " and {} samples.".format(self.n_quantiles, self.subsample)) - X = self._check_inputs(X, copy=False) + X = self._check_inputs(X, in_fit=True, copy=False) n_samples = X.shape[0] if self.n_quantiles > n_samples: @@ -2428,11 +2443,22 @@ def _transform_col(self, X_col, quantiles, inverse): return X_col - def _check_inputs(self, X, accept_sparse_negative=False, copy=False): + def _check_inputs(self, X, in_fit, accept_sparse_negative=False, + copy=False): """Check inputs before fit and transform""" - X = check_array(X, accept_sparse='csc', copy=copy, - dtype=FLOAT_DTYPES, - force_all_finite='allow-nan') + # In theory reset should be equal to `in_fit`, but there are tests + # checking the input number of feature and they expect a specific + # string, which is not the same one raised by check_n_features. So we + # don't check n_features_in_ here for now (it's done with adhoc code in + # the estimator anyway). + # TODO: set reset=in_fit when addressing reset in + # predict/transform/etc. + reset = True + + X = self._validate_data(X, reset=reset, + accept_sparse='csc', copy=copy, + dtype=FLOAT_DTYPES, + force_all_finite='allow-nan') # we only accept positive sparse matrix when ignore_implicit_zeros is # false and that we call fit or transform. with np.errstate(invalid='ignore'): # hide NaN comparison warnings @@ -2508,7 +2534,7 @@ def transform(self, X): Xt : ndarray or sparse matrix, shape (n_samples, n_features) The projected data. """ - X = self._check_inputs(X, copy=self.copy) + X = self._check_inputs(X, in_fit=False, copy=self.copy) self._check_is_fitted(X) return self._transform(X, inverse=False) @@ -2529,7 +2555,8 @@ def inverse_transform(self, X): Xt : ndarray or sparse matrix, shape (n_samples, n_features) The projected data. """ - X = self._check_inputs(X, accept_sparse_negative=True, copy=self.copy) + X = self._check_inputs(X, in_fit=False, accept_sparse_negative=True, + copy=self.copy) self._check_is_fitted(X) return self._transform(X, inverse=True) @@ -2543,7 +2570,7 @@ def quantile_transform(X, axis=0, n_quantiles=1000, ignore_implicit_zeros=False, subsample=int(1e5), random_state=None, - copy="warn"): + copy=True): """Transform features using quantiles information. This method transforms the features to follow a uniform or a normal @@ -2595,24 +2622,19 @@ def quantile_transform(X, axis=0, n_quantiles=1000, differ for value-identical sparse and dense matrices. random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by np.random. Note that this is used by subsampling and smoothing + Determines random number generation for subsampling and smoothing noise. + Please see ``subsample`` for more details. + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary ` - copy : boolean, optional, (default="warn") + copy : boolean, optional, (default=True) Set to False to perform inplace transformation and avoid a copy (if the input is already a numpy array). If True, a copy of `X` is transformed, leaving the original `X` unchanged - .. deprecated:: 0.21 - The default value of parameter `copy` will be changed from False - to True in 0.23. The current default of False is being changed to - make it more consistent with the default `copy` values of other - functions in :mod:`sklearn.preprocessing`. Furthermore, the - current default of False may have unexpected side effects by - modifying the value of `X` inplace + ..versionchnanged:: 0.23 + The default value of `copy` changed from False to True in 0.23. Returns ------- @@ -2649,17 +2671,6 @@ def quantile_transform(X, axis=0, n_quantiles=1000, see :ref:`examples/preprocessing/plot_all_scaling.py `. """ - if copy == "warn": - warnings.warn("The default value of `copy` will change from False to " - "True in 0.23 in order to make it more consistent with " - "the default `copy` values of other functions in " - ":mod:`sklearn.preprocessing` and prevent " - "unexpected side effects by modifying the value of `X` " - "inplace. To avoid inplace modifications of `X`, it is " - "recommended to explicitly set `copy=True`", - FutureWarning) - copy = False - n = QuantileTransformer(n_quantiles=n_quantiles, output_distribution=output_distribution, subsample=subsample, @@ -2695,6 +2706,8 @@ class PowerTransformer(TransformerMixin, BaseEstimator): Read more in the :ref:`User Guide `. + .. versionadded:: 0.20 + Parameters ---------- method : str, (default='yeo-johnson') @@ -2785,7 +2798,8 @@ def fit_transform(self, X, y=None): return self._fit(X, y, force_transform=True) def _fit(self, X, y=None, force_transform=False): - X = self._check_input(X, check_positive=True, check_method=True) + X = self._check_input(X, in_fit=True, check_positive=True, + check_method=True) if not self.copy and not force_transform: # if call from fit() X = X.copy() # force copy so that fit does not change X inplace @@ -2827,7 +2841,8 @@ def transform(self, X): The transformed data. """ check_is_fitted(self) - X = self._check_input(X, check_positive=True, check_shape=True) + X = self._check_input(X, in_fit=False, check_positive=True, + check_shape=True) transform_function = {'box-cox': boxcox, 'yeo-johnson': self._yeo_johnson_transform @@ -2873,7 +2888,7 @@ def inverse_transform(self, X): The original data """ check_is_fitted(self) - X = self._check_input(X, check_shape=True) + X = self._check_input(X, in_fit=False, check_shape=True) if self.standardize: X = self._scaler.inverse_transform(X) @@ -2978,7 +2993,7 @@ def _neg_log_likelihood(lmbda): # choosing bracket -2, 2 like for boxcox return optimize.brent(_neg_log_likelihood, brack=(-2, 2)) - def _check_input(self, X, check_positive=False, check_shape=False, + def _check_input(self, X, in_fit, check_positive=False, check_shape=False, check_method=False): """Validate the input before fit and transform. @@ -2996,8 +3011,8 @@ def _check_input(self, X, check_positive=False, check_shape=False, check_method : bool If True, check that the transformation method is valid. """ - X = check_array(X, ensure_2d=True, dtype=FLOAT_DTYPES, copy=self.copy, - force_all_finite='allow-nan') + X = self._validate_data(X, ensure_2d=True, dtype=FLOAT_DTYPES, + copy=self.copy, force_all_finite='allow-nan') with np.warnings.catch_warnings(): np.warnings.filterwarnings( @@ -3024,7 +3039,7 @@ def _more_tags(self): return {'allow_nan': True} -def power_transform(X, method='warn', standardize=True, copy=True): +def power_transform(X, method='yeo-johnson', standardize=True, copy=True): """ Power transforms are a family of parametric, monotonic transformations that are applied to make data more Gaussian-like. This is useful for @@ -3048,15 +3063,15 @@ def power_transform(X, method='warn', standardize=True, copy=True): X : array-like, shape (n_samples, n_features) The data to be transformed using a power transformation. - method : str + method : {'yeo-johnson', 'box-cox'}, default='yeo-johnson' The power transform method. Available methods are: - 'yeo-johnson' [1]_, works with positive and negative values - 'box-cox' [2]_, only works with strictly positive values - The default method will be changed from 'box-cox' to 'yeo-johnson' - in version 0.23. To suppress the FutureWarning, explicitly set the - parameter. + .. versionchanged:: 0.23 + The default value of the `method` parameter changed from + 'box-cox' to 'yeo-johnson' in 0.23. standardize : boolean, default=True Set to True to apply zero-mean, unit-variance normalization to the @@ -3108,12 +3123,5 @@ def power_transform(X, method='warn', standardize=True, copy=True): .. [2] G.E.P. Box and D.R. Cox, "An Analysis of Transformations", Journal of the Royal Statistical Society B, 26, 211-252 (1964). """ - if method == 'warn': - warnings.warn("The default value of 'method' will change from " - "'box-cox' to 'yeo-johnson' in version 0.23. Set " - "the 'method' argument explicitly to silence this " - "warning in the meantime.", - FutureWarning) - method = 'box-cox' pt = PowerTransformer(method=method, standardize=standardize, copy=copy) return pt.fit_transform(X) diff --git a/sklearn/preprocessing/_discretization.py b/sklearn/preprocessing/_discretization.py index 5a73bf5c7f845..67641601e06f5 100644 --- a/sklearn/preprocessing/_discretization.py +++ b/sklearn/preprocessing/_discretization.py @@ -137,7 +137,7 @@ def fit(self, X, y=None): ------- self """ - X = check_array(X, dtype='numeric') + X = self._validate_data(X, dtype='numeric') valid_encode = ('onehot', 'onehot-dense', 'ordinal') if self.encode not in valid_encode: diff --git a/sklearn/preprocessing/_encoders.py b/sklearn/preprocessing/_encoders.py index 73865d22605f8..86be9d335bd9e 100644 --- a/sklearn/preprocessing/_encoders.py +++ b/sklearn/preprocessing/_encoders.py @@ -171,6 +171,8 @@ class OneHotEncoder(_BaseEncoder): Read more in the :ref:`User Guide `. + .. versionchanged:: 0.20 + Parameters ---------- categories : 'auto' or a list of array-like, default='auto' @@ -184,7 +186,8 @@ class OneHotEncoder(_BaseEncoder): The used categories can be found in the ``categories_`` attribute. - drop : 'first' or a array-like of shape (n_features,), default=None + drop : {'first', 'if_binary'} or a array-like of shape (n_features,), \ + default=None Specifies a methodology to use to drop one of the categories per feature. This is useful in situations where perfectly collinear features cause problems, such as when feeding the resulting data @@ -193,6 +196,9 @@ class OneHotEncoder(_BaseEncoder): - None : retain all features (the default). - 'first' : drop the first category in each feature. If only one category is present, the feature will be dropped entirely. + - 'if_binary' : drop the first category in each feature with two + categories. Features with 1 or more than 2 categories are + left intact. - array : ``drop[i]`` is the category in feature ``X[:, i]`` that should be dropped. @@ -219,9 +225,13 @@ class OneHotEncoder(_BaseEncoder): (if any). drop_idx_ : array of shape (n_features,) - ``drop_idx_[i]`` is the index in ``categories_[i]`` of the category to - be dropped for each feature. None if all the transformed features will - be retained. + - ``drop_idx_[i]`` is the index in ``categories_[i]`` of the category + to be dropped for each feature. + - ``drop_idx_[i] = None`` if no category is to be dropped from the + feature with index ``i``, e.g. when `drop='if_binary'` and the + feature isn't binary. + - ``drop_idx_ = None`` if all the transformed features will be + retained. See Also -------- @@ -243,6 +253,9 @@ class OneHotEncoder(_BaseEncoder): values per feature and transform the data to a binary one-hot encoding. >>> from sklearn.preprocessing import OneHotEncoder + + One can discard categories not seen during `fit`: + >>> enc = OneHotEncoder(handle_unknown='ignore') >>> X = [['Male', 1], ['Female', 3], ['Female', 2]] >>> enc.fit(X) @@ -258,12 +271,22 @@ class OneHotEncoder(_BaseEncoder): >>> enc.get_feature_names(['gender', 'group']) array(['gender_Female', 'gender_Male', 'group_1', 'group_2', 'group_3'], dtype=object) + + One can always drop the first column for each feature: + >>> drop_enc = OneHotEncoder(drop='first').fit(X) >>> drop_enc.categories_ [array(['Female', 'Male'], dtype=object), array([1, 2, 3], dtype=object)] >>> drop_enc.transform([['Female', 1], ['Male', 2]]).toarray() array([[0., 0., 0.], [1., 1., 0.]]) + + Or drop a column for feature only having 2 categories: + + >>> drop_binary_enc = OneHotEncoder(drop='if_binary').fit(X) + >>> drop_binary_enc.transform([['Female', 1], ['Male', 2]]).toarray() + array([[0., 1., 0., 0.], + [1., 0., 1., 0.]]) """ def __init__(self, categories='auto', drop=None, sparse=True, @@ -291,15 +314,28 @@ def _validate_keywords(self): def _compute_drop_idx(self): if self.drop is None: return None - elif (isinstance(self.drop, str) and self.drop == 'first'): - return np.zeros(len(self.categories_), dtype=np.int_) - elif not isinstance(self.drop, str): + elif isinstance(self.drop, str): + if self.drop == 'first': + return np.zeros(len(self.categories_), dtype=np.object) + elif self.drop == 'if_binary': + return np.array([0 if len(cats) == 2 else None + for cats in self.categories_], dtype=np.object) + else: + msg = ( + "Wrong input for parameter `drop`. Expected " + "'first', 'if_binary', None or array of objects, got {}" + ) + raise ValueError(msg.format(type(self.drop))) + + else: try: self.drop = np.asarray(self.drop, dtype=object) droplen = len(self.drop) except (ValueError, TypeError): - msg = ("Wrong input for parameter `drop`. Expected " - "'first', None or array of objects, got {}") + msg = ( + "Wrong input for parameter `drop`. Expected " + "'first', 'if_binary', None or array of objects, got {}" + ) raise ValueError(msg.format(type(self.drop))) if droplen != len(self.categories_): msg = ("`drop` should have length equal to the number " @@ -318,11 +354,8 @@ def _compute_drop_idx(self): raise ValueError(msg) return np.array([np.where(cat_list == val)[0][0] for (val, cat_list) in - zip(self.drop, self.categories_)], dtype=np.int_) - else: - msg = ("Wrong input for parameter `drop`. Expected " - "'first', None or array of objects, got {}") - raise ValueError(msg.format(type(self.drop))) + zip(self.drop, self.categories_)], + dtype=np.object) def fit(self, X, y=None): """ @@ -389,26 +422,38 @@ def transform(self, X): n_samples, n_features = X_int.shape - if self.drop is not None: - to_drop = self.drop_idx_.reshape(1, -1) - + if self.drop_idx_ is not None: + to_drop = self.drop_idx_.copy() # We remove all the dropped categories from mask, and decrement all # categories that occur after them to avoid an empty column. - keep_cells = X_int != to_drop - X_mask &= keep_cells + n_values = [] + for i, cats in enumerate(self.categories_): + n_cats = len(cats) + + # drop='if_binary' but feature isn't binary + if to_drop[i] is None: + # set to cardinality to not drop from X_int + to_drop[i] = n_cats + n_values.append(n_cats) + else: # dropped + n_values.append(n_cats - 1) + + to_drop = to_drop.reshape(1, -1) X_int[X_int > to_drop] -= 1 - n_values = [len(cats) - 1 for cats in self.categories_] + X_mask &= keep_cells else: n_values = [len(cats) for cats in self.categories_] mask = X_mask.ravel() - n_values = np.array([0] + n_values) - feature_indices = np.cumsum(n_values) + feature_indices = np.cumsum([0] + n_values) indices = (X_int + feature_indices[:-1]).ravel()[mask] - indptr = X_mask.sum(axis=1).cumsum() - indptr = np.insert(indptr, 0, 0) - data = np.ones(n_samples * n_features)[mask] + + indptr = np.empty(n_samples + 1, dtype=np.int) + indptr[0] = 0 + np.sum(X_mask, axis=1, out=indptr[1:]) + np.cumsum(indptr[1:], out=indptr[1:]) + data = np.ones(indptr[-1]) out = sparse.csr_matrix((data, indices, indptr), shape=(n_samples, feature_indices[-1]), @@ -440,12 +485,14 @@ def inverse_transform(self, X): n_samples, _ = X.shape n_features = len(self.categories_) - if self.drop is None: + if self.drop_idx_ is None: n_transformed_features = sum(len(cats) for cats in self.categories_) else: - n_transformed_features = sum(len(cats) - 1 - for cats in self.categories_) + n_transformed_features = sum( + len(cats) - 1 if to_drop is not None else len(cats) + for cats, to_drop in zip(self.categories_, self.drop_idx_) + ) # validate shape of passed X msg = ("Shape of the passed X data is not correct. Expected {0} " @@ -461,7 +508,7 @@ def inverse_transform(self, X): found_unknown = {} for i in range(n_features): - if self.drop is None: + if self.drop_idx_ is None or self.drop_idx_[i] is None: cats = self.categories_[i] else: cats = np.delete(self.categories_[i], self.drop_idx_[i]) @@ -484,9 +531,9 @@ def inverse_transform(self, X): if unknown.any(): found_unknown[i] = unknown # drop will either be None or handle_unknown will be error. If - # self.drop is not None, then we can safely assume that all of + # self.drop_idx_ is not None, then we can safely assume that all of # the nulls in each column are the dropped value - elif self.drop is not None: + elif self.drop_idx_ is not None: dropped = np.asarray(sub.sum(axis=1) == 0).flatten() if dropped.any(): X_tr[dropped, i] = self.categories_[i][self.drop_idx_[i]] @@ -533,7 +580,7 @@ def get_feature_names(self, input_features=None): for i in range(len(cats)): names = [ input_features[i] + '_' + str(t) for t in cats[i]] - if self.drop is not None: + if self.drop_idx_ is not None and self.drop_idx_[i] is not None: names.pop(self.drop_idx_[i]) feature_names.extend(names) @@ -551,6 +598,8 @@ class OrdinalEncoder(_BaseEncoder): Read more in the :ref:`User Guide `. + .. versionchanged:: 0.20.1 + Parameters ---------- categories : 'auto' or a list of array-like, default='auto' diff --git a/sklearn/preprocessing/_function_transformer.py b/sklearn/preprocessing/_function_transformer.py index 4aceaa08100f2..85ce3a1f845c1 100644 --- a/sklearn/preprocessing/_function_transformer.py +++ b/sklearn/preprocessing/_function_transformer.py @@ -92,7 +92,7 @@ def __init__(self, func=None, inverse_func=None, validate=False, def _check_input(self, X): if self.validate: - return check_array(X, accept_sparse=self.accept_sparse) + return self._validate_data(X, accept_sparse=self.accept_sparse) return X def _check_inverse_transform(self, X): diff --git a/sklearn/preprocessing/_label.py b/sklearn/preprocessing/_label.py index 9fbc959969e33..c644aa919f5cf 100644 --- a/sklearn/preprocessing/_label.py +++ b/sklearn/preprocessing/_label.py @@ -176,6 +176,8 @@ class LabelEncoder(TransformerMixin, BaseEstimator): Read more in the :ref:`User Guide `. + .. versionadded:: 0.12 + Attributes ---------- classes_ : array of shape (n_class,) diff --git a/sklearn/preprocessing/tests/test_data.py b/sklearn/preprocessing/tests/test_data.py index 060719200fa99..7999df083631c 100644 --- a/sklearn/preprocessing/tests/test_data.py +++ b/sklearn/preprocessing/tests/test_data.py @@ -25,6 +25,7 @@ from sklearn.utils._testing import assert_allclose from sklearn.utils._testing import assert_allclose_dense_sparse from sklearn.utils._testing import skip_if_32bit +from sklearn.utils._testing import _convert_container from sklearn.utils.sparsefuncs import mean_variance_axis from sklearn.preprocessing._data import _handle_zeros_in_scale @@ -79,9 +80,7 @@ def toarray(a): def _check_dim_1axis(a): - if isinstance(a, list): - return np.array(a).shape[0] - return a.shape[0] + return np.asarray(a).shape[0] def assert_correct_incr(i, batch_start, batch_stop, n, chunk_size, @@ -1453,7 +1452,6 @@ def test_quantile_transform_sparse_toy(): assert_array_almost_equal(X.toarray(), X_trans_inv.toarray()) -@pytest.mark.filterwarnings("ignore: The default value of `copy`") # 0.23 def test_quantile_transform_axis1(): X = np.array([[0, 25, 50, 75, 100], [2, 4, 6, 8, 10], @@ -1533,16 +1531,24 @@ def test_quantile_transform_nan(): assert not np.isnan(transformer.quantiles_[:, 1:]).any() -def test_deprecated_quantile_transform_copy(): - future_message = ("The default value of `copy` will change from False to " - "True in 0.23 in order to make it more consistent with " - "the default `copy` values of other functions in " - ":mod:`sklearn.preprocessing` and prevent " - "unexpected side effects by modifying the value of `X` " - "inplace. To avoid inplace modifications of `X`, it is " - "recommended to explicitly set `copy=True`") - assert_warns_message(FutureWarning, future_message, quantile_transform, - np.array([[0, 1], [0, 0.5], [1, 0]])) +@pytest.mark.parametrize("array_type", ['array', 'sparse']) +def test_quantile_transformer_sorted_quantiles(array_type): + # Non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/15733 + # Taken from upstream bug report: + # https://github.com/numpy/numpy/issues/14685 + X = np.array([0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 1, 1, 9, 9, 9, 8, 8, 7] * 10) + X = 0.1 * X.reshape(-1, 1) + X = _convert_container(X, array_type) + + n_quantiles = 100 + qt = QuantileTransformer(n_quantiles=n_quantiles).fit(X) + + # Check that the estimated quantile threasholds are monotically + # increasing: + quantiles = qt.quantiles_[:, 0] + assert len(quantiles) == 100 + assert all(np.diff(quantiles) >= 0) def test_robust_scaler_invalid_range(): @@ -1941,7 +1947,7 @@ def test_normalizer_max(): X_norm2 = toarray(X_norm2) for X_norm in (X_norm1, X_norm2): - row_maxs = X_norm.max(axis=1) + row_maxs = abs(X_norm).max(axis=1) for i in range(3): assert_almost_equal(row_maxs[i], 1.0) assert_almost_equal(row_maxs[3], 0.0) @@ -1960,6 +1966,27 @@ def test_normalizer_max(): assert_almost_equal(la.norm(X_norm[3]), 0.0) +def test_normalizer_max_sign(): + # check that we normalize by a positive number even for negative data + rng = np.random.RandomState(0) + X_dense = rng.randn(4, 5) + # set the row number 3 to zero + X_dense[3, :] = 0.0 + # check for mixed data where the value with + # largest magnitude is negative + X_dense[2, abs(X_dense[2, :]).argmax()] *= -1 + X_all_neg = -np.abs(X_dense) + X_all_neg_sparse = sparse.csr_matrix(X_all_neg) + + for X in (X_dense, X_all_neg, X_all_neg_sparse): + normalizer = Normalizer(norm='max') + X_norm = normalizer.transform(X) + assert X_norm is not X + X_norm = toarray(X_norm) + assert_array_equal( + np.sign(X_norm), np.sign(toarray(X))) + + def test_normalize(): # Test normalize function # Only tests functionality not used by the tests for Normalizer. @@ -2163,7 +2190,6 @@ def test_fit_cold_start(): scaler.fit_transform(X_2d) -@pytest.mark.filterwarnings("ignore: The default value of `copy`") # 0.23 def test_quantile_transform_valid_axis(): X = np.array([[0, 25, 50, 75, 100], [2, 4, 6, 8, 10], @@ -2468,19 +2494,15 @@ def test_power_transformer_copy_False(method, standardize): assert X_trans is X_inv_trans -def test_power_transform_default_method(): - X = np.abs(X_2d) - - future_warning_message = ( - "The default value of 'method' " - "will change from 'box-cox'" - ) - assert_warns_message(FutureWarning, future_warning_message, - power_transform, X) - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - X_trans_default = power_transform(X) - - X_trans_boxcox = power_transform(X, method='box-cox') - assert_array_equal(X_trans_boxcox, X_trans_default) +@pytest.mark.parametrize( + "X_2", + [sparse.random(10, 1, density=0.8, random_state=0), + sparse.csr_matrix(np.full((10, 1), fill_value=np.nan))] +) +def test_standard_scaler_sparse_partial_fit_finite_variance(X_2): + # non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/16448 + X_1 = sparse.random(5, 1, density=0.8) + scaler = StandardScaler(with_mean=False) + scaler.fit(X_1).partial_fit(X_2) + assert np.isfinite(scaler.var_[0]) diff --git a/sklearn/preprocessing/tests/test_encoders.py b/sklearn/preprocessing/tests/test_encoders.py index 78590f40ffba5..7e23aa2d485c2 100644 --- a/sklearn/preprocessing/tests/test_encoders.py +++ b/sklearn/preprocessing/tests/test_encoders.py @@ -14,12 +14,6 @@ from sklearn.preprocessing import OrdinalEncoder -def toarray(a): - if hasattr(a, "toarray"): - a = a.toarray() - return a - - def test_one_hot_encoder_sparse_dense(): # check that sparse and dense will give the same results @@ -265,6 +259,31 @@ def test_one_hot_encoder_inverse(sparse_, drop): enc.inverse_transform(X_tr) +def test_one_hot_encoder_inverse_if_binary(): + X = np.array([['Male', 1], + ['Female', 3], + ['Female', 2]], dtype=object) + ohe = OneHotEncoder(drop='if_binary', sparse=False) + X_tr = ohe.fit_transform(X) + assert_array_equal(ohe.inverse_transform(X_tr), X) + + +# check that resetting drop option without refitting does not throw an error +@pytest.mark.parametrize('drop', ['if_binary', 'first', None]) +@pytest.mark.parametrize('reset_drop', ['if_binary', 'first', None]) +def test_one_hot_encoder_drop_reset(drop, reset_drop): + X = np.array([['Male', 1], + ['Female', 3], + ['Female', 2]], dtype=object) + ohe = OneHotEncoder(drop=drop, sparse=False) + ohe.fit(X) + X_tr = ohe.transform(X) + feature_names = ohe.get_feature_names() + ohe.set_params(drop=reset_drop) + assert_array_equal(ohe.inverse_transform(X_tr), X) + assert_allclose(ohe.transform(X), X_tr) + assert_array_equal(ohe.get_feature_names(), feature_names) + @pytest.mark.parametrize("method", ['fit', 'fit_transform']) @pytest.mark.parametrize("X", [ [1, 2], @@ -385,8 +404,9 @@ def test_one_hot_encoder_pandas(): @pytest.mark.parametrize("drop, expected_names", [('first', ['x0_c', 'x2_b']), + ('if_binary', ['x0_c', 'x1_2', 'x2_b']), (['c', 2, 'b'], ['x0_b', 'x2_a'])], - ids=['first', 'manual']) + ids=['first', 'binary', 'manual']) def test_one_hot_encoder_feature_names_drop(drop, expected_names): X = [['c', 2, 'a'], ['b', 2, 'b']] @@ -398,6 +418,36 @@ def test_one_hot_encoder_feature_names_drop(drop, expected_names): assert_array_equal(expected_names, feature_names) +def test_one_hot_encoder_drop_equals_if_binary(): + # Canonical case + X = [[10, 'yes'], + [20, 'no'], + [30, 'yes']] + expected = np.array([[1., 0., 0., 1.], + [0., 1., 0., 0.], + [0., 0., 1., 1.]]) + expected_drop_idx = np.array([None, 0]) + + ohe = OneHotEncoder(drop='if_binary', sparse=False) + result = ohe.fit_transform(X) + assert_array_equal(ohe.drop_idx_, expected_drop_idx) + assert_allclose(result, expected) + + # with only one cat, the behaviour is equivalent to drop=None + X = [['true', 'a'], + ['false', 'a'], + ['false', 'a']] + expected = np.array([[1., 1.], + [0., 1.], + [0., 1.]]) + expected_drop_idx = np.array([0, None]) + + ohe = OneHotEncoder(drop='if_binary', sparse=False) + result = ohe.fit_transform(X) + assert_array_equal(ohe.drop_idx_, expected_drop_idx) + assert_allclose(result, expected) + + @pytest.mark.parametrize("X", [np.array([[1, np.nan]]).T, np.array([['a', np.nan]], dtype=object).T], ids=['numeric', 'object']) @@ -629,9 +679,9 @@ def test_categories(density, drop): for drop_cat, drop_idx, cat_list in zip(drop, ohe_test.drop_idx_, ohe_test.categories_): - assert cat_list[drop_idx] == drop_cat + assert cat_list[int(drop_idx)] == drop_cat assert isinstance(ohe_test.drop_idx_, np.ndarray) - assert ohe_test.drop_idx_.dtype == np.int_ + assert ohe_test.drop_idx_.dtype == np.object @pytest.mark.parametrize('Encoder', [OneHotEncoder, OrdinalEncoder]) diff --git a/sklearn/preprocessing/tests/test_label.py b/sklearn/preprocessing/tests/test_label.py index 69c025fcc76e3..887fa90c98d61 100644 --- a/sklearn/preprocessing/tests/test_label.py +++ b/sklearn/preprocessing/tests/test_label.py @@ -14,6 +14,7 @@ from sklearn.utils._testing import assert_array_equal from sklearn.utils._testing import assert_warns_message from sklearn.utils._testing import ignore_warnings +from sklearn.utils import _to_object_array from sklearn.preprocessing._label import LabelBinarizer from sklearn.preprocessing._label import MultiLabelBinarizer @@ -222,7 +223,7 @@ def test_label_encoder_negative_ints(): def test_label_encoder_str_bad_shape(dtype): le = LabelEncoder() le.fit(np.array(["apple", "orange"], dtype=dtype)) - msg = "bad input shape" + msg = "should be a 1d array" with pytest.raises(ValueError, match=msg): le.transform("apple") @@ -245,7 +246,7 @@ def test_label_encoder_errors(): le.inverse_transform([-2, -3, -4]) # Fail on inverse_transform("") - msg = "bad input shape ()" + msg = r"should be a 1d array.+shape \(\)" with pytest.raises(ValueError, match=msg): le.inverse_transform("") @@ -433,8 +434,7 @@ def test_multilabel_binarizer_same_length_sequence(): def test_multilabel_binarizer_non_integer_labels(): - tuple_classes = np.empty(3, dtype=object) - tuple_classes[:] = [(1,), (2,), (3,)] + tuple_classes = _to_object_array([(1,), (2,), (3,)]) inputs = [ ([('2', '3'), ('1',), ('1', '2')], ['1', '2', '3']), ([('b', 'c'), ('a',), ('a', 'b')], ['a', 'b', 'c']), diff --git a/sklearn/random_projection.py b/sklearn/random_projection.py index aa47437ba1cc3..d18f3bf846901 100644 --- a/sklearn/random_projection.py +++ b/sklearn/random_projection.py @@ -176,11 +176,10 @@ def _gaussian_random_matrix(n_components, n_features, random_state=None): Dimensionality of the original source space. random_state : int, RandomState instance or None, optional (default=None) - Control the pseudo random number generator used to generate the matrix - at fit time. If int, random_state is the seed used by the random - number generator; If RandomState instance, random_state is the random - number generator; If None, the random number generator is the - RandomState instance used by `np.random`. + Controls the pseudo random number generator used to generate the matrix + at fit time. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Returns ------- @@ -243,11 +242,10 @@ def _sparse_random_matrix(n_components, n_features, density='auto', Achlioptas, 2001. random_state : int, RandomState instance or None, optional (default=None) - Control the pseudo random number generator used to generate the matrix - at fit time. If int, random_state is the seed used by the random - number generator; If RandomState instance, random_state is the random - number generator; If None, the random number generator is the - RandomState instance used by `np.random`. + Controls the pseudo random number generator used to generate the matrix + at fit time. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Returns ------- @@ -356,7 +354,7 @@ def fit(self, X, y=None): self """ - X = check_array(X, accept_sparse=['csr', 'csc']) + X = self._validate_data(X, accept_sparse=['csr', 'csc']) n_samples, n_features = X.shape @@ -462,11 +460,10 @@ class GaussianRandomProjection(BaseRandomProjection): dimensions (n_components) in the target projection space. random_state : int, RandomState instance or None, optional (default=None) - Control the pseudo random number generator used to generate the matrix - at fit time. If int, random_state is the seed used by the random - number generator; If RandomState instance, random_state is the random - number generator; If None, the random number generator is the - RandomState instance used by `np.random`. + Controls the pseudo random number generator used to generate the + projection matrix at fit time. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Attributes ---------- @@ -584,11 +581,10 @@ class SparseRandomProjection(BaseRandomProjection): the input is sparse. random_state : int, RandomState instance or None, optional (default=None) - Control the pseudo random number generator used to generate the matrix - at fit time. If int, random_state is the seed used by the random - number generator; If RandomState instance, random_state is the random - number generator; If None, the random number generator is the - RandomState instance used by `np.random`. + Controls the pseudo random number generator used to generate the + projection matrix at fit time. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Attributes ---------- diff --git a/sklearn/semi_supervised/_label_propagation.py b/sklearn/semi_supervised/_label_propagation.py index 0ec687aae7d20..a84a9950aa3ac 100644 --- a/sklearn/semi_supervised/_label_propagation.py +++ b/sklearn/semi_supervised/_label_propagation.py @@ -76,29 +76,29 @@ class BaseLabelPropagation(ClassifierMixin, BaseEstimator, metaclass=ABCMeta): Parameters ---------- - kernel : {'knn', 'rbf', callable} + kernel : {'knn', 'rbf'} or callable, default='rbf' String identifier for kernel function to use or the kernel function itself. Only 'rbf' and 'knn' strings are valid inputs. The function - passed should take two inputs, each of shape [n_samples, n_features], - and return a [n_samples, n_samples] shaped weight matrix + passed should take two inputs, each of shape (n_samples, n_features), + and return a (n_samples, n_samples) shaped weight matrix. - gamma : float - Parameter for rbf kernel + gamma : float, default=20 + Parameter for rbf kernel. - n_neighbors : integer > 0 - Parameter for knn kernel + n_neighbors : int, default=7 + Parameter for knn kernel. Need to be strictly positive. - alpha : float - Clamping factor + alpha : float, default=1.0 + Clamping factor. - max_iter : integer - Change maximum number of iterations allowed + max_iter : int, default=30 + Change maximum number of iterations allowed. - tol : float + tol : float, default=1e-3 Convergence tolerance: threshold to consider the system at steady - state + state. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of parallel jobs to run. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` @@ -158,11 +158,12 @@ def predict(self, X): Parameters ---------- X : array-like of shape (n_samples, n_features) + The data matrix. Returns ------- - y : array_like, shape = [n_samples] - Predictions for input data + y : ndarray of shape (n_samples,) + Predictions for input data. """ probas = self.predict_proba(X) return self.classes_[np.argmax(probas, axis=1)].ravel() @@ -177,12 +178,13 @@ def predict_proba(self, X): Parameters ---------- X : array-like of shape (n_samples, n_features) + The data matrix. Returns ------- - probabilities : array, shape = [n_samples, n_classes] + probabilities : ndarray of shape (n_samples, n_classes) Normalized probability distributions across - class labels + class labels. """ check_is_fitted(self) @@ -195,7 +197,8 @@ class labels for weight_matrix in weight_matrices]) else: weight_matrices = weight_matrices.T - probabilities = np.dot(weight_matrices, self.label_distributions_) + probabilities = safe_sparse_dot( + weight_matrices, self.label_distributions_) normalizer = np.atleast_2d(np.sum(probabilities, axis=1)).T probabilities /= normalizer return probabilities @@ -210,17 +213,17 @@ def fit(self, X, y): Parameters ---------- X : array-like of shape (n_samples, n_features) - A {n_samples by n_samples} size matrix will be created from this + A matrix of shape (n_samples, n_samples) will be created from this. - y : array_like, shape = [n_samples] - n_labeled_samples (unlabeled points are marked as -1) - All unlabeled samples will be transductively assigned labels + y : array-like of shape (n_samples,) + `n_labeled_samples` (unlabeled points are marked as -1) + All unlabeled samples will be transductively assigned labels. Returns ------- - self : returns an instance of self. + self : object """ - X, y = check_X_y(X, y) + X, y = self._validate_data(X, y) self.X_ = X check_classification_targets(y) @@ -289,6 +292,7 @@ def fit(self, X, y): self.n_iter_ += 1 normalizer = np.sum(self.label_distributions_, axis=1)[:, np.newaxis] + normalizer[normalizer == 0] = 1 self.label_distributions_ /= normalizer # set the transduction item @@ -305,26 +309,26 @@ class LabelPropagation(BaseLabelPropagation): Parameters ---------- - kernel : {'knn', 'rbf', callable} + kernel : {'knn', 'rbf'} or callable, default='rbf' String identifier for kernel function to use or the kernel function itself. Only 'rbf' and 'knn' strings are valid inputs. The function - passed should take two inputs, each of shape [n_samples, n_features], - and return a [n_samples, n_samples] shaped weight matrix. + passed should take two inputs, each of shape (n_samples, n_features), + and return a (n_samples, n_samples) shaped weight matrix. - gamma : float - Parameter for rbf kernel + gamma : float, default=20 + Parameter for rbf kernel. - n_neighbors : integer > 0 - Parameter for knn kernel + n_neighbors : int, default=7 + Parameter for knn kernel which need to be strictly positive. - max_iter : integer - Change maximum number of iterations allowed + max_iter : int, default=1000 + Change maximum number of iterations allowed. - tol : float + tol : float, 1e-3 Convergence tolerance: threshold to consider the system at steady - state + state. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of parallel jobs to run. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` @@ -332,16 +336,16 @@ class LabelPropagation(BaseLabelPropagation): Attributes ---------- - X_ : array, shape = [n_samples, n_features] + X_ : ndarray of shape (n_samples, n_features) Input array. - classes_ : array, shape = [n_classes] + classes_ : ndarray of shape (n_classes,) The distinct labels used in classifying instances. - label_distributions_ : array, shape = [n_samples, n_classes] + label_distributions_ : ndarray of shape (n_samples, n_classes) Categorical distribution for each item. - transduction_ : array, shape = [n_samples] + transduction_ : ndarray of shape (n_samples) Label assigned to each item via the transduction. n_iter_ : int @@ -411,33 +415,33 @@ class LabelSpreading(BaseLabelPropagation): Parameters ---------- - kernel : {'knn', 'rbf', callable} + kernel : {'knn', 'rbf'} or callable, default='rbf' String identifier for kernel function to use or the kernel function itself. Only 'rbf' and 'knn' strings are valid inputs. The function - passed should take two inputs, each of shape [n_samples, n_features], - and return a [n_samples, n_samples] shaped weight matrix + passed should take two inputs, each of shape (n_samples, n_features), + and return a (n_samples, n_samples) shaped weight matrix. - gamma : float - parameter for rbf kernel + gamma : float, default=20 + Parameter for rbf kernel. - n_neighbors : integer > 0 - parameter for knn kernel + n_neighbors : int, default=7 + Parameter for knn kernel which is a strictly positive integer. - alpha : float + alpha : float, default=0.2 Clamping factor. A value in (0, 1) that specifies the relative amount that an instance should adopt the information from its neighbors as opposed to its initial label. alpha=0 means keeping the initial label information; alpha=1 means replacing all initial information. - max_iter : integer - maximum number of iterations allowed + max_iter : int, default=30 + Maximum number of iterations allowed. - tol : float + tol : float, default=1e-3 Convergence tolerance: threshold to consider the system at steady - state + state. - n_jobs : int or None, optional (default=None) + n_jobs : int, default=None The number of parallel jobs to run. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` @@ -445,16 +449,16 @@ class LabelSpreading(BaseLabelPropagation): Attributes ---------- - X_ : array, shape = [n_samples, n_features] + X_ : ndarray of shape (n_samples, n_features) Input array. - classes_ : array, shape = [n_classes] + classes_ : ndarray of shape (n_classes,) The distinct labels used in classifying instances. - label_distributions_ : array, shape = [n_samples, n_classes] + label_distributions_ : ndarray of shape (n_samples, n_classes) Categorical distribution for each item. - transduction_ : array, shape = [n_samples] + transduction_ : ndarray of shape (n_samples,) Label assigned to each item via the transduction. n_iter_ : int diff --git a/sklearn/semi_supervised/tests/test_label_propagation.py b/sklearn/semi_supervised/tests/test_label_propagation.py index 7e20350b20b2f..015f6fa191853 100644 --- a/sklearn/semi_supervised/tests/test_label_propagation.py +++ b/sklearn/semi_supervised/tests/test_label_propagation.py @@ -3,10 +3,13 @@ import numpy as np import pytest +from scipy.sparse import issparse from sklearn.utils._testing import assert_warns from sklearn.utils._testing import assert_no_warnings from sklearn.semi_supervised import _label_propagation as label_propagation from sklearn.metrics.pairwise import rbf_kernel +from sklearn.model_selection import train_test_split +from sklearn.neighbors import NearestNeighbors from sklearn.datasets import make_classification from sklearn.exceptions import ConvergenceWarning from numpy.testing import assert_array_almost_equal @@ -152,3 +155,51 @@ def test_convergence_warning(): mdl = label_propagation.LabelPropagation(kernel='rbf', max_iter=500) assert_no_warnings(mdl.fit, X, y) + + +def test_label_propagation_non_zero_normalizer(): + # check that we don't divide by zero in case of null normalizer + # non-regression test for + # https://github.com/scikit-learn/scikit-learn/pull/15946 + X = np.array([[100., 100.], [100., 100.], [0., 0.], [0., 0.]]) + y = np.array([0, 1, -1, -1]) + mdl = label_propagation.LabelSpreading(kernel='knn', + max_iter=100, + n_neighbors=1) + assert_no_warnings(mdl.fit, X, y) + + +def test_predict_sparse_callable_kernel(): + # This is a non-regression test for #15866 + + # Custom sparse kernel (top-K RBF) + def topk_rbf(X, Y=None, n_neighbors=10, gamma=1e-5): + nn = NearestNeighbors(n_neighbors=10, metric='euclidean', n_jobs=-1) + nn.fit(X) + W = -1 * nn.kneighbors_graph(Y, mode='distance').power(2) * gamma + np.exp(W.data, out=W.data) + assert issparse(W) + return W.T + + n_classes = 4 + n_samples = 500 + n_test = 10 + X, y = make_classification(n_classes=n_classes, + n_samples=n_samples, + n_features=20, + n_informative=20, + n_redundant=0, + n_repeated=0, + random_state=0) + + X_train, X_test, y_train, y_test = train_test_split(X, y, + test_size=n_test, + random_state=0) + + model = label_propagation.LabelSpreading(kernel=topk_rbf) + model.fit(X_train, y_train) + assert model.score(X_test, y_test) >= 0.9 + + model = label_propagation.LabelPropagation(kernel=topk_rbf) + model.fit(X_train, y_train) + assert model.score(X_test, y_test) >= 0.9 diff --git a/sklearn/setup.py b/sklearn/setup.py index cc257c30e6f43..e759cdabc88ee 100644 --- a/sklearn/setup.py +++ b/sklearn/setup.py @@ -53,6 +53,8 @@ def configuration(parent_package='', top_path=None): config.add_subpackage('experimental/tests') config.add_subpackage('ensemble/_hist_gradient_boosting') config.add_subpackage('ensemble/_hist_gradient_boosting/tests') + config.add_subpackage('_loss/') + config.add_subpackage('_loss/tests') # submodules which have their own setup.py config.add_subpackage('cluster') diff --git a/sklearn/svm/_base.py b/sklearn/svm/_base.py index d327e0fef26e4..662a4ffa24678 100644 --- a/sklearn/svm/_base.py +++ b/sklearn/svm/_base.py @@ -14,7 +14,8 @@ from ..utils import compute_class_weight from ..utils.extmath import safe_sparse_dot from ..utils.validation import check_is_fitted, _check_large_sparse -from ..utils.validation import _check_sample_weight +from ..utils.validation import _num_samples +from ..utils.validation import _check_sample_weight, check_consistent_length from ..utils.multiclass import check_classification_targets from ..exceptions import ConvergenceWarning from ..exceptions import NotFittedError @@ -74,7 +75,7 @@ def __init__(self, kernel, degree, gamma, coef0, tol, C, nu, epsilon, shrinking, probability, cache_size, class_weight, verbose, max_iter, random_state): - if self._impl not in LIBSVM_IMPL: # pragma: no cover + if self._impl not in LIBSVM_IMPL: raise ValueError("impl should be one of %s, %s was given" % ( LIBSVM_IMPL, self._impl)) @@ -109,17 +110,17 @@ def fit(self, X, y, sample_weight=None): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training vectors, where n_samples is the number of samples and n_features is the number of features. For kernel="precomputed", the expected shape of X is (n_samples, n_samples). - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) Target values (class labels in classification, real numbers in regression) - sample_weight : array-like, shape (n_samples,) + sample_weight : array-like of shape (n_samples,), default=None Per-sample weights. Rescale C per sample. Higher weights force the classifier to put more emphasis on these points. @@ -143,9 +144,13 @@ def fit(self, X, y, sample_weight=None): raise TypeError("Sparse precomputed kernels are not supported.") self._sparse = sparse and not callable(self.kernel) - X, y = check_X_y(X, y, dtype=np.float64, - order='C', accept_sparse='csr', - accept_large_sparse=False) + if callable(self.kernel): + check_consistent_length(X, y) + else: + X, y = self._validate_data(X, y, dtype=np.float64, + order='C', accept_sparse='csr', + accept_large_sparse=False) + y = self._validate_targets(y) sample_weight = np.asarray([] @@ -154,24 +159,31 @@ def fit(self, X, y, sample_weight=None): solver_type = LIBSVM_IMPL.index(self._impl) # input validation - if solver_type != 2 and X.shape[0] != y.shape[0]: + n_samples = _num_samples(X) + if solver_type != 2 and n_samples != y.shape[0]: raise ValueError("X and y have incompatible shapes.\n" + "X has %s samples, but y has %s." % - (X.shape[0], y.shape[0])) + (n_samples, y.shape[0])) - if self.kernel == "precomputed" and X.shape[0] != X.shape[1]: + if self.kernel == "precomputed" and n_samples != X.shape[1]: raise ValueError("Precomputed matrix must be a square matrix." " Input is a {}x{} matrix." .format(X.shape[0], X.shape[1])) - if sample_weight.shape[0] > 0 and sample_weight.shape[0] != X.shape[0]: + if sample_weight.shape[0] > 0 and sample_weight.shape[0] != n_samples: raise ValueError("sample_weight and X have incompatible shapes: " "%r vs %r\n" "Note: Sparse matrices cannot be indexed w/" "boolean masks (use `indices=True` in CV)." % (sample_weight.shape, X.shape)) - if isinstance(self.gamma, str): + kernel = 'precomputed' if callable(self.kernel) else self.kernel + + if kernel == 'precomputed': + # unused but needs to be a float for cython code that ignores + # it anyway + self._gamma = 0. + elif isinstance(self.gamma, str): if self.gamma == 'scale': # var = E[X^2] - E[X]^2 if sparse X_var = ((X.multiply(X)).mean() - (X.mean()) ** 2 @@ -187,19 +199,15 @@ def fit(self, X, y, sample_weight=None): else: self._gamma = self.gamma - kernel = self.kernel - if callable(kernel): - kernel = 'precomputed' - fit = self._sparse_fit if self._sparse else self._dense_fit - if self.verbose: # pragma: no cover + if self.verbose: print('[LibSVM]', end='') seed = rnd.randint(np.iinfo('i').max) fit(X, y, sample_weight, solver_type, kernel, random_seed=seed) # see comment on the other call to np.iinfo in this file - self.shape_fit_ = X.shape + self.shape_fit_ = X.shape if hasattr(X, "shape") else (n_samples, ) # In binary case, we need to flip the sign of coef, intercept and # decision function. Use self._intercept_ and self._dual_coef_ @@ -246,8 +254,8 @@ def _dense_fit(self, X, y, sample_weight, solver_type, kernel, # we don't pass **self.get_params() to allow subclasses to # add other parameters to __init__ self.support_, self.support_vectors_, self._n_support, \ - self.dual_coef_, self.intercept_, self.probA_, \ - self.probB_, self.fit_status_ = libsvm.fit( + self.dual_coef_, self.intercept_, self._probA, \ + self._probB, self.fit_status_ = libsvm.fit( X, y, svm_type=solver_type, sample_weight=sample_weight, class_weight=self.class_weight_, kernel=kernel, C=self.C, @@ -270,7 +278,7 @@ def _sparse_fit(self, X, y, sample_weight, solver_type, kernel, self.support_, self.support_vectors_, dual_coef_data, \ self.intercept_, self._n_support, \ - self.probA_, self.probB_, self.fit_status_ = \ + self._probA, self._probB, self.fit_status_ = \ libsvm_sparse.libsvm_sparse_train( X.shape[1], X.data, X.indices, X.indptr, y, solver_type, kernel_type, self.degree, self._gamma, self.coef0, self.tol, @@ -304,13 +312,13 @@ def predict(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) For kernel="precomputed", the expected shape of X is (n_samples_test, n_samples_train). Returns ------- - y_pred : array, shape (n_samples,) + y_pred : ndarray of shape (n_samples,) """ X = self._validate_for_predict(X) predict = self._sparse_predict if self._sparse else self._dense_predict @@ -334,7 +342,7 @@ def _dense_predict(self, X): return libsvm.predict( X, self.support_, self.support_vectors_, self._n_support, self._dual_coef_, self._intercept_, - self.probA_, self.probB_, svm_type=svm_type, kernel=kernel, + self._probA, self._probB, svm_type=svm_type, kernel=kernel, degree=self.degree, coef0=self.coef0, gamma=self._gamma, cache_size=self.cache_size) @@ -359,7 +367,7 @@ def _sparse_predict(self, X): C, self.class_weight_, self.nu, self.epsilon, self.shrinking, self.probability, self._n_support, - self.probA_, self.probB_) + self._probA, self._probB) def _compute_kernel(self, X): """Return the data transformed by a callable kernel""" @@ -377,11 +385,11 @@ def _decision_function(self, X): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Returns ------- - X : array-like, shape (n_samples, n_class * (n_class-1) / 2) + X : array-like of shape (n_samples, n_class * (n_class-1) / 2) Returns the decision function of the sample for each class in the model. """ @@ -413,7 +421,7 @@ def _dense_decision_function(self, X): return libsvm.decision_function( X, self.support_, self.support_vectors_, self._n_support, self._dual_coef_, self._intercept_, - self.probA_, self.probB_, + self._probA, self._probB, svm_type=LIBSVM_IMPL.index(self._impl), kernel=kernel, degree=self.degree, cache_size=self.cache_size, coef0=self.coef0, gamma=self._gamma) @@ -438,13 +446,15 @@ def _sparse_decision_function(self, X): self.C, self.class_weight_, self.nu, self.epsilon, self.shrinking, self.probability, self._n_support, - self.probA_, self.probB_) + self._probA, self._probB) def _validate_for_predict(self, X): check_is_fitted(self) - X = check_array(X, accept_sparse='csr', dtype=np.float64, order="C", - accept_large_sparse=False) + if not callable(self.kernel): + X = check_array(X, accept_sparse='csr', dtype=np.float64, + order="C", accept_large_sparse=False) + if self._sparse and not sp.isspmatrix(X): X = sp.csr_matrix(X) if self._sparse: @@ -454,17 +464,16 @@ def _validate_for_predict(self, X): raise ValueError( "cannot use sparse input in %r trained on dense data" % type(self).__name__) - n_samples, n_features = X.shape if self.kernel == "precomputed": if X.shape[1] != self.shape_fit_[0]: raise ValueError("X.shape[1] = %d should be equal to %d, " "the number of samples at training time" % (X.shape[1], self.shape_fit_[0])) - elif n_features != self.shape_fit_[1]: + elif not callable(self.kernel) and X.shape[1] != self.shape_fit_[1]: raise ValueError("X.shape[1] = %d should be equal to %d, " "the number of features at training time" % - (n_features, self.shape_fit_[1])) + (X.shape[1], self.shape_fit_[1])) return X @property @@ -539,11 +548,11 @@ def decision_function(self, X): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Returns ------- - X : array-like, shape (n_samples, n_classes * (n_classes-1) / 2) + X : ndarray of shape (n_samples, n_classes * (n_classes-1) / 2) Returns the decision function of the sample for each class in the model. If decision_function_shape='ovr', the shape is (n_samples, @@ -572,13 +581,14 @@ def predict(self, X): Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) or \ + (n_samples_test, n_samples_train) For kernel="precomputed", the expected shape of X is - [n_samples_test, n_samples_train] + (n_samples_test, n_samples_train). Returns ------- - y_pred : array, shape (n_samples,) + y_pred : ndarray of shape (n_samples,) Class labels for samples in X. """ check_is_fitted(self) @@ -615,13 +625,13 @@ def predict_proba(self): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) For kernel="precomputed", the expected shape of X is [n_samples_test, n_samples_train] Returns ------- - T : array-like, shape (n_samples, n_classes) + T : ndarray of shape (n_samples, n_classes) Returns the probability of the sample for each class in the model. The columns correspond to the classes in sorted order, as they appear in the attribute :term:`classes_`. @@ -654,13 +664,14 @@ def predict_log_proba(self): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) or \ + (n_samples_test, n_samples_train) For kernel="precomputed", the expected shape of X is - [n_samples_test, n_samples_train] + (n_samples_test, n_samples_train). Returns ------- - T : array-like, shape (n_samples, n_classes) + T : ndarray of shape (n_samples, n_classes) Returns the log-probabilities of the sample for each class in the model. The columns correspond to the classes in sorted order, as they appear in the attribute :term:`classes_`. @@ -689,7 +700,7 @@ def _dense_predict_proba(self, X): pprob = libsvm.predict_proba( X, self.support_, self.support_vectors_, self._n_support, self._dual_coef_, self._intercept_, - self.probA_, self.probB_, + self._probA, self._probB, svm_type=svm_type, kernel=kernel, degree=self.degree, cache_size=self.cache_size, coef0=self.coef0, gamma=self._gamma) @@ -715,7 +726,7 @@ def _sparse_predict_proba(self, X): self.C, self.class_weight_, self.nu, self.epsilon, self.shrinking, self.probability, self._n_support, - self.probA_, self.probB_) + self._probA, self._probB) def _get_coef(self): if self.dual_coef_.shape[0] == 1: @@ -732,6 +743,14 @@ def _get_coef(self): return coef + @property + def probA_(self): + return self._probA + + @property + def probB_(self): + return self._probB + def _get_liblinear_solver_type(multi_class, penalty, loss, dual): """Find the liblinear magic number for the solver. @@ -747,7 +766,7 @@ def _get_liblinear_solver_type(multi_class, penalty, loss, dual): """ # nested dicts containing level 1: available loss functions, # level2: available penalties for the given loss function, - # level3: wether the dual solver is available for the specified + # level3: whether the dual solver is available for the specified # combination of loss function and penalty _solver_type_dict = { 'logistic_regression': { @@ -804,11 +823,11 @@ def _fit_liblinear(X, y, C, fit_intercept, intercept_scaling, class_weight, Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training vector, where n_samples in the number of samples and n_features is the number of features. - y : array-like, shape (n_samples,) + y : array-like of shape (n_samples,) Target vector relative to X C : float @@ -825,7 +844,7 @@ def _fit_liblinear(X, y, C, fit_intercept, intercept_scaling, class_weight, In order to avoid this, one should increase the intercept_scaling. such that the feature vector becomes [x, intercept_scaling]. - class_weight : {dict, 'balanced'}, optional + class_weight : dict or 'balanced', default=None Weights associated with classes in the form ``{class_label: weight}``. If not given, all classes are supposed to have weight one. For multi-output problems, a list of dicts can be provided in the same @@ -835,7 +854,7 @@ def _fit_liblinear(X, y, C, fit_intercept, intercept_scaling, class_weight, weights inversely proportional to class frequencies in the input data as ``n_samples / (n_classes * np.bincount(y))`` - penalty : str, {'l1', 'l2'} + penalty : {'l1', 'l2'} The norm of the penalty used in regularization. dual : bool @@ -850,14 +869,12 @@ def _fit_liblinear(X, y, C, fit_intercept, intercept_scaling, class_weight, tol : float Stopping condition. - random_state : int, RandomState instance or None, optional (default=None) - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. + random_state : int or RandomState instance, default=None + Controls the pseudo random number generation for shuffling the data. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. - multi_class : str, {'ovr', 'crammer_singer'} + multi_class : {'ovr', 'crammer_singer'}, default='ovr' `ovr` trains n_classes one-vs-rest classifiers, while `crammer_singer` optimizes a joint objective over all classes. While `crammer_singer` is interesting from an theoretical perspective @@ -866,21 +883,22 @@ def _fit_liblinear(X, y, C, fit_intercept, intercept_scaling, class_weight, If `crammer_singer` is chosen, the options loss, penalty and dual will be ignored. - loss : str, {'logistic_regression', 'hinge', 'squared_hinge', - 'epsilon_insensitive', 'squared_epsilon_insensitive} + loss : {'logistic_regression', 'hinge', 'squared_hinge', \ + 'epsilon_insensitive', 'squared_epsilon_insensitive}, \ + default='logistic_regression' The loss function used to fit the model. - epsilon : float, optional (default=0.1) + epsilon : float, default=0.1 Epsilon parameter in the epsilon-insensitive loss function. Note that the value of this parameter depends on the scale of the target variable y. If unsure, set epsilon=0. - sample_weight : array-like, optional + sample_weight : array-like of shape (n_samples,), default=None Weights assigned to each sample. Returns ------- - coef_ : ndarray, shape (n_features, n_features + 1) + coef_ : ndarray of shape (n_features, n_features + 1) The coefficient vector got by minimizing the objective function. intercept_ : float @@ -911,8 +929,8 @@ def _fit_liblinear(X, y, C, fit_intercept, intercept_scaling, class_weight, bias = -1.0 if fit_intercept: if intercept_scaling <= 0: - raise ValueError("Intercept scaling is %r but needs to be greater than 0." - " To disable fitting an intercept," + raise ValueError("Intercept scaling is %r but needs to be greater " + "than 0. To disable fitting an intercept," " set fit_intercept=False." % intercept_scaling) else: bias = intercept_scaling diff --git a/sklearn/svm/_bounds.py b/sklearn/svm/_bounds.py index c60f0cd033213..1e1ed8939ce5f 100644 --- a/sklearn/svm/_bounds.py +++ b/sklearn/svm/_bounds.py @@ -21,23 +21,23 @@ def l1_min_c(X, y, loss='squared_hinge', fit_intercept=True, Parameters ---------- - X : {array-like or sparse matrix} of shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Training vector, where n_samples in the number of samples and n_features is the number of features. - y : array, shape = [n_samples] - Target vector relative to X + y : array-like of shape (n_samples,) + Target vector relative to X. - loss : {'squared_hinge', 'log'}, default 'squared_hinge' + loss : {'squared_hinge', 'log'}, default='squared_hinge' Specifies the loss function. With 'squared_hinge' it is the squared hinge loss (a.k.a. L2 loss). With 'log' it is the loss of logistic regression models. - fit_intercept : bool, default: True + fit_intercept : bool, default=True Specifies if the intercept should be fitted by the model. It must match the fit() method parameter. - intercept_scaling : float, default: 1 + intercept_scaling : float, default=1.0 when fit_intercept is True, instance vector x becomes [x, intercept_scaling], i.e. a "synthetic" feature with constant value equals to diff --git a/sklearn/svm/_classes.py b/sklearn/svm/_classes.py index 50c2356142ae2..fbaa6e97ec616 100644 --- a/sklearn/svm/_classes.py +++ b/sklearn/svm/_classes.py @@ -8,6 +8,7 @@ from ..utils import check_X_y from ..utils.validation import _num_samples from ..utils.multiclass import check_classification_targets +from ..utils.deprecation import deprecated class LinearSVC(BaseEstimator, LinearClassifierMixin, @@ -26,28 +27,28 @@ class LinearSVC(BaseEstimator, LinearClassifierMixin, Parameters ---------- - penalty : str, 'l1' or 'l2' (default='l2') + penalty : {'l1', 'l2'}, default='l2' Specifies the norm used in the penalization. The 'l2' penalty is the standard used in SVC. The 'l1' leads to ``coef_`` vectors that are sparse. - loss : str, 'hinge' or 'squared_hinge' (default='squared_hinge') + loss : {'hinge', 'squared_hinge'}, default='squared_hinge' Specifies the loss function. 'hinge' is the standard SVM loss (used e.g. by the SVC class) while 'squared_hinge' is the square of the hinge loss. - dual : bool, (default=True) + dual : bool, default=True Select the algorithm to either solve the dual or primal optimization problem. Prefer dual=False when n_samples > n_features. - tol : float, optional (default=1e-4) + tol : float, default=1e-4 Tolerance for stopping criteria. - C : float, optional (default=1.0) + C : float, default=1.0 Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive. - multi_class : str, 'ovr' or 'crammer_singer' (default='ovr') + multi_class : {'ovr', 'crammer_singer'}, default='ovr' Determines the multi-class strategy if `y` contains more than two classes. ``"ovr"`` trains n_classes one-vs-rest classifiers, while @@ -58,12 +59,12 @@ class LinearSVC(BaseEstimator, LinearClassifierMixin, If ``"crammer_singer"`` is chosen, the options loss, penalty and dual will be ignored. - fit_intercept : bool, optional (default=True) + fit_intercept : bool, default=True Whether to calculate the intercept for this model. If set to false, no intercept will be used in calculations (i.e. data is expected to be already centered). - intercept_scaling : float, optional (default=1) + intercept_scaling : float, default=1 When self.fit_intercept is True, instance vector x becomes ``[x, self.intercept_scaling]``, i.e. a "synthetic" feature with constant value equals to @@ -74,7 +75,7 @@ class LinearSVC(BaseEstimator, LinearClassifierMixin, To lessen the effect of regularization on synthetic feature weight (and therefore on the intercept) intercept_scaling has to be increased. - class_weight : {dict, 'balanced'}, optional + class_weight : dict or 'balanced', default=None Set the parameter C of class i to ``class_weight[i]*C`` for SVC. If not given, all classes are supposed to have weight one. @@ -82,38 +83,36 @@ class LinearSVC(BaseEstimator, LinearClassifierMixin, weights inversely proportional to class frequencies in the input data as ``n_samples / (n_classes * np.bincount(y))``. - verbose : int, (default=0) + verbose : int, default=0 Enable verbose output. Note that this setting takes advantage of a per-process runtime setting in liblinear that, if enabled, may not work properly in a multithreaded context. - random_state : int, RandomState instance or None, optional (default=None) - The seed of the pseudo random number generator to use when shuffling - the data for the dual coordinate descent (if ``dual=True``). When - ``dual=False`` the underlying implementation of :class:`LinearSVC` - is not random and ``random_state`` has no effect on the results. If - int, random_state is the seed used by the random number generator; If - RandomState instance, random_state is the random number generator; If - None, the random number generator is the RandomState instance used by - `np.random`. - - max_iter : int, (default=1000) + random_state : int or RandomState instance, default=None + Controls the pseudo random number generation for shuffling the data for + the dual coordinate descent (if ``dual=True``). When ``dual=False`` the + underlying implementation of :class:`LinearSVC` is not random and + ``random_state`` has no effect on the results. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. + + max_iter : int, default=1000 The maximum number of iterations to be run. Attributes ---------- - coef_ : array, shape = [1, n_features] if n_classes == 2 \ -else [n_classes, n_features] + coef_ : ndarray of shape (1, n_features) if n_classes == 2 \ + else (n_classes, n_features) Weights assigned to the features (coefficients in the primal problem). This is only available in the case of a linear kernel. ``coef_`` is a readonly property derived from ``raw_coef_`` that follows the internal memory layout of liblinear. - intercept_ : array, shape = [1] if n_classes == 2 else [n_classes] + intercept_ : ndarray of shape (1,) if n_classes == 2 else (n_classes,) Constants in decision function. - classes_ : array of shape (n_classes,) + classes_ : ndarray of shape (n_classes,) The unique classes labels. n_iter_ : int @@ -214,25 +213,13 @@ def fit(self, X, y, sample_weight=None): self : object An instance of the estimator. """ - # FIXME Remove l1/l2 support in 0.23 ---------------------------------- - msg = ("loss='%s' has been deprecated in favor of " - "loss='%s' as of 0.16. Backward compatibility" - " for the loss='%s' will be removed in %s") - - if self.loss in ('l1', 'l2'): - old_loss = self.loss - self.loss = {'l1': 'hinge', 'l2': 'squared_hinge'}.get(self.loss) - warnings.warn(msg % (old_loss, self.loss, old_loss, '0.23'), - FutureWarning) - # --------------------------------------------------------------------- - if self.C < 0: raise ValueError("Penalty term must be positive; got (C=%r)" % self.C) - X, y = check_X_y(X, y, accept_sparse='csr', - dtype=np.float64, order="C", - accept_large_sparse=False) + X, y = self._validate_data(X, y, accept_sparse='csr', + dtype=np.float64, order="C", + accept_large_sparse=False) check_classification_targets(y) self.classes_ = np.unique(y) @@ -267,29 +254,30 @@ class LinearSVR(RegressorMixin, LinearModel): Parameters ---------- - epsilon : float, optional (default=0.0) + epsilon : float, default=0.0 Epsilon parameter in the epsilon-insensitive loss function. Note that the value of this parameter depends on the scale of the target variable y. If unsure, set ``epsilon=0``. - tol : float, optional (default=1e-4) + tol : float, default=1e-4 Tolerance for stopping criteria. - C : float, optional (default=1.0) + C : float, default=1.0 Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive. - loss : string, optional (default='epsilon_insensitive') + loss : {'epsilon_insensitive', 'squared_epsilon_insensitive'}, \ + default='epsilon_insensitive' Specifies the loss function. The epsilon-insensitive loss (standard SVR) is the L1 loss, while the squared epsilon-insensitive loss ('squared_epsilon_insensitive') is the L2 loss. - fit_intercept : boolean, optional (default=True) + fit_intercept : bool, default=True Whether to calculate the intercept for this model. If set to false, no intercept will be used in calculations (i.e. data is expected to be already centered). - intercept_scaling : float, optional (default=1) + intercept_scaling : float, default=1. When self.fit_intercept is True, instance vector x becomes [x, self.intercept_scaling], i.e. a "synthetic" feature with constant value equals to @@ -300,35 +288,34 @@ class LinearSVR(RegressorMixin, LinearModel): To lessen the effect of regularization on synthetic feature weight (and therefore on the intercept) intercept_scaling has to be increased. - dual : bool, (default=True) + dual : bool, default=True Select the algorithm to either solve the dual or primal optimization problem. Prefer dual=False when n_samples > n_features. - verbose : int, (default=0) + verbose : int, default=0 Enable verbose output. Note that this setting takes advantage of a per-process runtime setting in liblinear that, if enabled, may not work properly in a multithreaded context. - random_state : int, RandomState instance or None, optional (default=None) - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. + random_state : int or RandomState instance, default=None + Controls the pseudo random number generation for shuffling the data. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. - max_iter : int, (default=1000) + max_iter : int, default=1000 The maximum number of iterations to be run. Attributes ---------- - coef_ : array, shape = [n_features] if n_classes == 2 else [n_classes, n_features] + coef_ : ndarray of shape (n_features) if n_classes == 2 \ + else (n_classes, n_features) Weights assigned to the features (coefficients in the primal problem). This is only available in the case of a linear kernel. `coef_` is a readonly property derived from `raw_coef_` that follows the internal memory layout of liblinear. - intercept_ : array, shape = [1] if n_classes == 2 else [n_classes] + intercept_ : ndarray of shape (1) if n_classes == 2 else (n_classes) Constants in decision function. n_iter_ : int @@ -402,28 +389,15 @@ def fit(self, X, y, sample_weight=None): Returns ------- self : object + An instance of the estimator. """ - # FIXME Remove l1/l2 support in 0.23 ---------------------------------- - msg = ("loss='%s' has been deprecated in favor of " - "loss='%s' as of 0.16. Backward compatibility" - " for the loss='%s' will be removed in %s") - - if self.loss in ('l1', 'l2'): - old_loss = self.loss - self.loss = {'l1': 'epsilon_insensitive', - 'l2': 'squared_epsilon_insensitive' - }.get(self.loss) - warnings.warn(msg % (old_loss, self.loss, old_loss, '0.23'), - FutureWarning) - # --------------------------------------------------------------------- - if self.C < 0: raise ValueError("Penalty term must be positive; got (C=%r)" % self.C) - X, y = check_X_y(X, y, accept_sparse='csr', - dtype=np.float64, order="C", - accept_large_sparse=False) + X, y = self._validate_data(X, y, accept_sparse='csr', + dtype=np.float64, order="C", + accept_large_sparse=False) penalty = 'l2' # SVR only accepts l2 penalty self.coef_, self.intercept_, self.n_iter_ = _fit_liblinear( X, y, self.C, self.fit_intercept, self.intercept_scaling, @@ -456,12 +430,12 @@ class SVC(BaseSVC): Parameters ---------- - C : float, optional (default=1.0) + C : float, default=1.0 Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive. The penalty is a squared l2 penalty. - kernel : string, optional (default='rbf') + kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'}, default='rbf' Specifies the kernel type to be used in the algorithm. It must be one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed' or a callable. @@ -469,11 +443,11 @@ class SVC(BaseSVC): used to pre-compute the kernel matrix from data matrices; that matrix should be an array of shape ``(n_samples, n_samples)``. - degree : int, optional (default=3) + degree : int, default=3 Degree of the polynomial kernel function ('poly'). Ignored by all other kernels. - gamma : {'scale', 'auto'} or float, optional (default='scale') + gamma : {'scale', 'auto'} or float, default='scale' Kernel coefficient for 'rbf', 'poly' and 'sigmoid'. - if ``gamma='scale'`` (default) is passed then it uses @@ -483,26 +457,26 @@ class SVC(BaseSVC): .. versionchanged:: 0.22 The default value of ``gamma`` changed from 'auto' to 'scale'. - coef0 : float, optional (default=0.0) + coef0 : float, default=0.0 Independent term in kernel function. It is only significant in 'poly' and 'sigmoid'. - shrinking : boolean, optional (default=True) + shrinking : bool, default=True Whether to use the shrinking heuristic. - probability : boolean, optional (default=False) + probability : bool, default=False Whether to enable probability estimates. This must be enabled prior to calling `fit`, will slow down that method as it internally uses 5-fold cross-validation, and `predict_proba` may be inconsistent with `predict`. Read more in the :ref:`User Guide `. - tol : float, optional (default=1e-3) + tol : float, default=1e-3 Tolerance for stopping criterion. - cache_size : float, optional + cache_size : float, default=200 Specify the size of the kernel cache (in MB). - class_weight : {dict, 'balanced'}, optional + class_weight : dict or 'balanced', default=None Set the parameter C of class i to class_weight[i]*C for SVC. If not given, all classes are supposed to have weight one. @@ -510,15 +484,15 @@ class SVC(BaseSVC): weights inversely proportional to class frequencies in the input data as ``n_samples / (n_classes * np.bincount(y))`` - verbose : bool, default: False + verbose : bool, default=False Enable verbose output. Note that this setting takes advantage of a per-process runtime setting in libsvm that, if enabled, may not work properly in a multithreaded context. - max_iter : int, optional (default=-1) + max_iter : int, default=-1 Hard limit on iterations within solver, or -1 for no limit. - decision_function_shape : 'ovo', 'ovr', default='ovr' + decision_function_shape : {'ovo', 'ovr'}, default='ovr' Whether to return a one-vs-rest ('ovr') decision function of shape (n_samples, n_classes) as all other classifiers, or the original one-vs-one ('ovo') decision function of libsvm which has shape @@ -534,7 +508,7 @@ class SVC(BaseSVC): .. versionchanged:: 0.17 Deprecated *decision_function_shape='ovo' and None*. - break_ties : bool, optional (default=False) + break_ties : bool, default=False If true, ``decision_function_shape='ovr'``, and number of classes > 2, :term:`predict` will break ties according to the confidence values of :term:`decision_function`; otherwise the first class among the tied @@ -543,32 +517,31 @@ class SVC(BaseSVC): .. versionadded:: 0.22 - random_state : int, RandomState instance or None, optional (default=None) - The seed of the pseudo random number generator used when shuffling - the data for probability estimates. If int, random_state is the - seed used by the random number generator; If RandomState instance, - random_state is the random number generator; If None, the random - number generator is the RandomState instance used by `np.random`. + random_state : int or RandomState instance, default=None + Controls the pseudo random number generation for shuffling the data for + probability estimates. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Attributes ---------- - support_ : array-like of shape (n_SV) + support_ : ndarray of shape (n_SV,) Indices of support vectors. - support_vectors_ : array-like of shape (n_SV, n_features) + support_vectors_ : ndarray of shape (n_SV, n_features) Support vectors. - n_support_ : array-like, dtype=int32, shape = [n_class] + n_support_ : ndarray of shape (n_class,), dtype=int32 Number of support vectors for each class. - dual_coef_ : array, shape = [n_class-1, n_SV] + dual_coef_ : ndarray of shape (n_class-1, n_SV) Coefficients of the support vector in the decision function. For multiclass, coefficient for all 1-vs-1 classifiers. The layout of the coefficients in the multiclass case is somewhat non-trivial. See the section about multi-class classification in the SVM section of the User Guide for details. - coef_ : array, shape = [n_class * (n_class-1) / 2, n_features] + coef_ : ndarray of shape (n_class * (n_class-1) / 2, n_features) Weights assigned to the features (coefficients in the primal problem). This is only available in the case of a linear kernel. @@ -581,11 +554,11 @@ class SVC(BaseSVC): fit_status_ : int 0 if correctly fitted, 1 otherwise (will raise warning) - classes_ : array of shape (n_classes,) + classes_ : ndarray of shape (n_classes,) The classes labels. - probA_ : array, shape = [n_class * (n_class-1) / 2] - probB_ : array, shape = [n_class * (n_class-1) / 2] + probA_ : ndarray of shape (n_class * (n_class-1) / 2) + probB_ : ndarray of shape (n_class * (n_class-1) / 2) If `probability=True`, it corresponds to the parameters learned in Platt scaling to produce probability estimates from decision values. If `probability=False`, it's an empty array. Platt scaling uses the @@ -665,23 +638,23 @@ class NuSVC(BaseSVC): Parameters ---------- - nu : float, optional (default=0.5) + nu : float, default=0.5 An upper bound on the fraction of training errors and a lower bound of the fraction of support vectors. Should be in the interval (0, 1]. - kernel : string, optional (default='rbf') + kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'}, default='rbf' Specifies the kernel type to be used in the algorithm. It must be one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed' or a callable. If none is given, 'rbf' will be used. If a callable is given it is used to precompute the kernel matrix. - degree : int, optional (default=3) + degree : int, default=3 Degree of the polynomial kernel function ('poly'). Ignored by all other kernels. - gamma : {'scale', 'auto'} or float, optional (default='scale') + gamma : {'scale', 'auto'} or float, default='scale' Kernel coefficient for 'rbf', 'poly' and 'sigmoid'. - if ``gamma='scale'`` (default) is passed then it uses @@ -691,41 +664,41 @@ class NuSVC(BaseSVC): .. versionchanged:: 0.22 The default value of ``gamma`` changed from 'auto' to 'scale'. - coef0 : float, optional (default=0.0) + coef0 : float, default=0.0 Independent term in kernel function. It is only significant in 'poly' and 'sigmoid'. - shrinking : boolean, optional (default=True) + shrinking : bool, default=True Whether to use the shrinking heuristic. - probability : boolean, optional (default=False) + probability : bool, default=False Whether to enable probability estimates. This must be enabled prior to calling `fit`, will slow down that method as it internally uses 5-fold cross-validation, and `predict_proba` may be inconsistent with `predict`. Read more in the :ref:`User Guide `. - tol : float, optional (default=1e-3) + tol : float, default=1e-3 Tolerance for stopping criterion. - cache_size : float, optional + cache_size : float, default=200 Specify the size of the kernel cache (in MB). - class_weight : {dict, 'balanced'}, optional + class_weight : {dict, 'balanced'}, default=None Set the parameter C of class i to class_weight[i]*C for SVC. If not given, all classes are supposed to have weight one. The "balanced" mode uses the values of y to automatically adjust weights inversely proportional to class frequencies as ``n_samples / (n_classes * np.bincount(y))`` - verbose : bool, default: False + verbose : bool, default=False Enable verbose output. Note that this setting takes advantage of a per-process runtime setting in libsvm that, if enabled, may not work properly in a multithreaded context. - max_iter : int, optional (default=-1) + max_iter : int, default=-1 Hard limit on iterations within solver, or -1 for no limit. - decision_function_shape : 'ovo', 'ovr', default='ovr' + decision_function_shape : {'ovo', 'ovr'}, default='ovr' Whether to return a one-vs-rest ('ovr') decision function of shape (n_samples, n_classes) as all other classifiers, or the original one-vs-one ('ovo') decision function of libsvm which has shape @@ -740,7 +713,7 @@ class NuSVC(BaseSVC): .. versionchanged:: 0.17 Deprecated *decision_function_shape='ovo' and None*. - break_ties : bool, optional (default=False) + break_ties : bool, default=False If true, ``decision_function_shape='ovr'``, and number of classes > 2, :term:`predict` will break ties according to the confidence values of :term:`decision_function`; otherwise the first class among the tied @@ -749,32 +722,31 @@ class NuSVC(BaseSVC): .. versionadded:: 0.22 - random_state : int, RandomState instance or None, optional (default=None) - The seed of the pseudo random number generator used when shuffling - the data for probability estimates. If int, random_state is the seed - used by the random number generator; If RandomState instance, - random_state is the random number generator; If None, the random - number generator is the RandomState instance used by `np.random`. + random_state : int or RandomState instance, default=None + Controls the pseudo random number generation for shuffling the data for + probability estimates. + Pass an int for reproducible output across multiple function calls. + See :term:`Glossary `. Attributes ---------- - support_ : array-like of shape (n_SV) + support_ : ndarray of shape (n_SV,) Indices of support vectors. - support_vectors_ : array-like of shape (n_SV, n_features) + support_vectors_ : ndarray of shape (n_SV, n_features) Support vectors. - n_support_ : array-like, dtype=int32, shape = [n_class] + n_support_ : ndarray of shape (n_class), dtype=int32 Number of support vectors for each class. - dual_coef_ : array, shape = [n_class-1, n_SV] + dual_coef_ : ndarray of shape (n_class-1, n_SV) Coefficients of the support vector in the decision function. For multiclass, coefficient for all 1-vs-1 classifiers. The layout of the coefficients in the multiclass case is somewhat non-trivial. See the section about multi-class classification in the SVM section of the User Guide for details. - coef_ : array, shape = [n_class * (n_class-1) / 2, n_features] + coef_ : ndarray of shape (n_class * (n_class-1) / 2, n_features) Weights assigned to the features (coefficients in the primal problem). This is only available in the case of a linear kernel. @@ -784,13 +756,13 @@ class NuSVC(BaseSVC): intercept_ : ndarray of shape (n_class * (n_class-1) / 2,) Constants in decision function. - classes_ : array of shape (n_classes,) + classes_ : ndarray of shape (n_classes,) The unique classes labels. fit_status_ : int 0 if correctly fitted, 1 if the algorithm did not converge. - probA_ : ndarray, shape of (n_class * (n_class-1) / 2,) + probA_ : ndarray of shape (n_class * (n_class-1) / 2,) probB_ : ndarray of shape (n_class * (n_class-1) / 2,) If `probability=True`, it corresponds to the parameters learned in Platt scaling to produce probability estimates from decision values. @@ -856,6 +828,15 @@ def __init__(self, nu=0.5, kernel='rbf', degree=3, gamma='scale', break_ties=break_ties, random_state=random_state) + def _more_tags(self): + return { + '_xfail_test': { + 'check_methods_subset_invariance': + 'fails for the decision_function method', + 'check_class_weight_classifiers': 'class_weight is ignored.' + } + } + class SVR(RegressorMixin, BaseLibSVM): """Epsilon-Support Vector Regression. @@ -873,18 +854,18 @@ class SVR(RegressorMixin, BaseLibSVM): Parameters ---------- - kernel : string, optional (default='rbf') + kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'}, default='rbf' Specifies the kernel type to be used in the algorithm. It must be one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed' or a callable. If none is given, 'rbf' will be used. If a callable is given it is used to precompute the kernel matrix. - degree : int, optional (default=3) + degree : int, default=3 Degree of the polynomial kernel function ('poly'). Ignored by all other kernels. - gamma : {'scale', 'auto'} or float, optional (default='scale') + gamma : {'scale', 'auto'} or float, default='scale' Kernel coefficient for 'rbf', 'poly' and 'sigmoid'. - if ``gamma='scale'`` (default) is passed then it uses @@ -894,50 +875,50 @@ class SVR(RegressorMixin, BaseLibSVM): .. versionchanged:: 0.22 The default value of ``gamma`` changed from 'auto' to 'scale'. - coef0 : float, optional (default=0.0) + coef0 : float, default=0.0 Independent term in kernel function. It is only significant in 'poly' and 'sigmoid'. - tol : float, optional (default=1e-3) + tol : float, default=1e-3 Tolerance for stopping criterion. - C : float, optional (default=1.0) + C : float, default=1.0 Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive. The penalty is a squared l2 penalty. - epsilon : float, optional (default=0.1) + epsilon : float, default=0.1 Epsilon in the epsilon-SVR model. It specifies the epsilon-tube within which no penalty is associated in the training loss function with points predicted within a distance epsilon from the actual value. - shrinking : boolean, optional (default=True) + shrinking : bool, default=True Whether to use the shrinking heuristic. - cache_size : float, optional + cache_size : float, default=200 Specify the size of the kernel cache (in MB). - verbose : bool, default: False + verbose : bool, default=False Enable verbose output. Note that this setting takes advantage of a per-process runtime setting in libsvm that, if enabled, may not work properly in a multithreaded context. - max_iter : int, optional (default=-1) + max_iter : int, default=-1 Hard limit on iterations within solver, or -1 for no limit. Attributes ---------- - support_ : array-like of shape (n_SV) + support_ : ndarray of shape (n_SV,) Indices of support vectors. - support_vectors_ : array-like of shape (n_SV, n_features) + support_vectors_ : ndarray of shape (n_SV, n_features) Support vectors. - dual_coef_ : array, shape = [1, n_SV] + dual_coef_ : ndarray of shape (1, n_SV) Coefficients of the support vector in the decision function. - coef_ : array, shape = [1, n_features] + coef_ : ndarray of shape (1, n_features) Weights assigned to the features (coefficients in the primal problem). This is only available in the case of a linear kernel. @@ -947,7 +928,7 @@ class SVR(RegressorMixin, BaseLibSVM): fit_status_ : int 0 if correctly fitted, 1 otherwise (will raise warning) - intercept_ : array, shape = [1] + intercept_ : ndarray of shape (1,) Constants in decision function. Examples @@ -958,8 +939,8 @@ class SVR(RegressorMixin, BaseLibSVM): >>> rng = np.random.RandomState(0) >>> y = rng.randn(n_samples) >>> X = rng.randn(n_samples, n_features) - >>> clf = SVR(C=1.0, epsilon=0.2) - >>> clf.fit(X, y) + >>> regr = SVR(C=1.0, epsilon=0.2) + >>> regr.fit(X, y) SVR(epsilon=0.2) See also @@ -991,6 +972,20 @@ def __init__(self, kernel='rbf', degree=3, gamma='scale', shrinking=shrinking, probability=False, cache_size=cache_size, class_weight=None, max_iter=max_iter, random_state=None) + @deprecated( + "The probA_ attribute is deprecated in version 0.23 and will be " + "removed in version 0.25.") + @property + def probA_(self): + return self._probA + + @deprecated( + "The probB_ attribute is deprecated in version 0.23 and will be " + "removed in version 0.25.") + @property + def probB_(self): + return self._probB + class NuSVR(RegressorMixin, BaseLibSVM): """Nu Support Vector Regression. @@ -1005,26 +1000,26 @@ class NuSVR(RegressorMixin, BaseLibSVM): Parameters ---------- - nu : float, optional + nu : float, default=0.5 An upper bound on the fraction of training errors and a lower bound of the fraction of support vectors. Should be in the interval (0, 1]. By default 0.5 will be taken. - C : float, optional (default=1.0) + C : float, default=1.0 Penalty parameter C of the error term. - kernel : string, optional (default='rbf') + kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'}, default='rbf' Specifies the kernel type to be used in the algorithm. It must be one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed' or a callable. If none is given, 'rbf' will be used. If a callable is given it is used to precompute the kernel matrix. - degree : int, optional (default=3) + degree : int, default=3 Degree of the polynomial kernel function ('poly'). Ignored by all other kernels. - gamma : {'scale', 'auto'} or float, optional (default='scale') + gamma : {'scale', 'auto'} or float, default='scale' Kernel coefficient for 'rbf', 'poly' and 'sigmoid'. - if ``gamma='scale'`` (default) is passed then it uses @@ -1034,46 +1029,46 @@ class NuSVR(RegressorMixin, BaseLibSVM): .. versionchanged:: 0.22 The default value of ``gamma`` changed from 'auto' to 'scale'. - coef0 : float, optional (default=0.0) + coef0 : float, default=0.0 Independent term in kernel function. It is only significant in 'poly' and 'sigmoid'. - shrinking : boolean, optional (default=True) + shrinking : bool, default=True Whether to use the shrinking heuristic. - tol : float, optional (default=1e-3) + tol : float, default=1e-3 Tolerance for stopping criterion. - cache_size : float, optional + cache_size : float, default=200 Specify the size of the kernel cache (in MB). - verbose : bool, default: False + verbose : bool, default=False Enable verbose output. Note that this setting takes advantage of a per-process runtime setting in libsvm that, if enabled, may not work properly in a multithreaded context. - max_iter : int, optional (default=-1) + max_iter : int, default=-1 Hard limit on iterations within solver, or -1 for no limit. Attributes ---------- - support_ : array-like of shape (n_SV) + support_ : ndarray of shape (n_SV,) Indices of support vectors. - support_vectors_ : array-like of shape (n_SV, n_features) + support_vectors_ : ndarray of shape (n_SV, n_features) Support vectors. - dual_coef_ : array, shape = [1, n_SV] + dual_coef_ : ndarray of shape (1, n_SV) Coefficients of the support vector in the decision function. - coef_ : array, shape = [1, n_features] + coef_ : ndarray of shape (1, n_features) Weights assigned to the features (coefficients in the primal problem). This is only available in the case of a linear kernel. `coef_` is readonly property derived from `dual_coef_` and `support_vectors_`. - intercept_ : array, shape = [1] + intercept_ : ndarray of shape (1,) Constants in decision function. Examples @@ -1084,8 +1079,8 @@ class NuSVR(RegressorMixin, BaseLibSVM): >>> np.random.seed(0) >>> y = np.random.randn(n_samples) >>> X = np.random.randn(n_samples, n_features) - >>> clf = NuSVR(C=1.0, nu=0.1) - >>> clf.fit(X, y) + >>> regr = NuSVR(C=1.0, nu=0.1) + >>> regr.fit(X, y) NuSVR(nu=0.1) See also @@ -1128,18 +1123,18 @@ class OneClassSVM(OutlierMixin, BaseLibSVM): Parameters ---------- - kernel : string, optional (default='rbf') + kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'}, default='rbf' Specifies the kernel type to be used in the algorithm. It must be one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed' or a callable. If none is given, 'rbf' will be used. If a callable is given it is used to precompute the kernel matrix. - degree : int, optional (default=3) + degree : int, default=3 Degree of the polynomial kernel function ('poly'). Ignored by all other kernels. - gamma : {'scale', 'auto'} or float, optional (default='scale') + gamma : {'scale', 'auto'} or float, default='scale' Kernel coefficient for 'rbf', 'poly' and 'sigmoid'. - if ``gamma='scale'`` (default) is passed then it uses @@ -1149,52 +1144,52 @@ class OneClassSVM(OutlierMixin, BaseLibSVM): .. versionchanged:: 0.22 The default value of ``gamma`` changed from 'auto' to 'scale'. - coef0 : float, optional (default=0.0) + coef0 : float, default=0.0 Independent term in kernel function. It is only significant in 'poly' and 'sigmoid'. - tol : float, optional + tol : float, default=1e-3 Tolerance for stopping criterion. - nu : float, optional + nu : float, default=0.5 An upper bound on the fraction of training errors and a lower bound of the fraction of support vectors. Should be in the interval (0, 1]. By default 0.5 will be taken. - shrinking : boolean, optional + shrinking : bool, default=True Whether to use the shrinking heuristic. - cache_size : float, optional + cache_size : float, default=200 Specify the size of the kernel cache (in MB). - verbose : bool, default: False + verbose : bool, default=False Enable verbose output. Note that this setting takes advantage of a per-process runtime setting in libsvm that, if enabled, may not work properly in a multithreaded context. - max_iter : int, optional (default=-1) + max_iter : int, default=-1 Hard limit on iterations within solver, or -1 for no limit. Attributes ---------- - support_ : array-like of shape (n_SV) + support_ : ndarray of shape (n_SV,) Indices of support vectors. - support_vectors_ : array-like of shape (n_SV, n_features) + support_vectors_ : ndarray of shape (n_SV, n_features) Support vectors. - dual_coef_ : array, shape = [1, n_SV] + dual_coef_ : ndarray of shape (1, n_SV) Coefficients of the support vectors in the decision function. - coef_ : array, shape = [1, n_features] + coef_ : ndarray of shape (1, n_features) Weights assigned to the features (coefficients in the primal problem). This is only available in the case of a linear kernel. `coef_` is readonly property derived from `dual_coef_` and `support_vectors_` - intercept_ : array, shape = [1,] + intercept_ : ndarray of shape (1,) Constant in the decision function. offset_ : float @@ -1229,16 +1224,15 @@ def __init__(self, kernel='rbf', degree=3, gamma='scale', random_state=None) def fit(self, X, y=None, sample_weight=None, **params): - """ - Detects the soft boundary of the set of samples X. + """Detects the soft boundary of the set of samples X. Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Set of samples, where n_samples is the number of samples and n_features is the number of features. - sample_weight : array-like, shape (n_samples,) + sample_weight : array-like of shape (n_samples,), default=None Per-sample weights. Rescale C per sample. Higher weights force the classifier to put more emphasis on these points. @@ -1266,11 +1260,12 @@ def decision_function(self, X): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) + The data matrix. Returns ------- - dec : array-like, shape (n_samples,) + dec : ndarray of shape (n_samples,) Returns the decision function of the samples. """ dec = self._decision_function(X).ravel() @@ -1281,31 +1276,46 @@ def score_samples(self, X): Parameters ---------- - X : array-like, shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) + The data matrix. Returns ------- - score_samples : array-like, shape (n_samples,) + score_samples : ndarray of shape (n_samples,) Returns the (unshifted) scoring function of the samples. """ return self.decision_function(X) + self.offset_ def predict(self, X): - """ - Perform classification on samples in X. + """Perform classification on samples in X. For a one-class model, +1 or -1 is returned. Parameters ---------- - X : {array-like, sparse matrix}, shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) or \ + (n_samples_test, n_samples_train) For kernel="precomputed", the expected shape of X is - [n_samples_test, n_samples_train] + (n_samples_test, n_samples_train). Returns ------- - y_pred : array, shape (n_samples,) + y_pred : ndarray of shape (n_samples,) Class labels for samples in X. """ y = super().predict(X) return np.asarray(y, dtype=np.intp) + + @deprecated( + "The probA_ attribute is deprecated in version 0.23 and will be " + "removed in version 0.25.") + @property + def probA_(self): + return self._probA + + @deprecated( + "The probB_ attribute is deprecated in version 0.23 and will be " + "removed in version 0.25.") + @property + def probB_(self): + return self._probB diff --git a/sklearn/svm/_liblinear.pxd b/sklearn/svm/_liblinear.pxi similarity index 98% rename from sklearn/svm/_liblinear.pxd rename to sklearn/svm/_liblinear.pxi index 0f10e54a532fe..148bf694dab4f 100644 --- a/sklearn/svm/_liblinear.pxd +++ b/sklearn/svm/_liblinear.pxi @@ -1,6 +1,3 @@ -cimport numpy as np - - cdef extern from "_cython_blas_helpers.h": ctypedef double (*dot_func)(int, double*, int, double*, int) ctypedef void (*axpy_func)(int, double, double*, int, double*, int) @@ -12,6 +9,7 @@ cdef extern from "_cython_blas_helpers.h": scal_func scal nrm2_func nrm2 + cdef extern from "linear.h": cdef struct feature_node cdef struct problem @@ -28,6 +26,7 @@ cdef extern from "linear.h": void free_and_destroy_model (model **) void destroy_param (parameter *) + cdef extern from "liblinear_helper.c": void copy_w(void *, model *, int) parameter *set_parameter(int, double, double, int, char *, char *, int, int, double) diff --git a/sklearn/svm/_liblinear.pyx b/sklearn/svm/_liblinear.pyx index 2f042748d94a0..9dd15e0716c7f 100644 --- a/sklearn/svm/_liblinear.pyx +++ b/sklearn/svm/_liblinear.pyx @@ -9,6 +9,8 @@ cimport numpy as np from ..utils._cython_blas cimport _dot, _axpy, _scal, _nrm2 +include "_liblinear.pxi" + np.import_array() diff --git a/sklearn/svm/_libsvm.pxd b/sklearn/svm/_libsvm.pxi similarity index 99% rename from sklearn/svm/_libsvm.pxd rename to sklearn/svm/_libsvm.pxi index 2664a335a372f..a3c8f1c33dd1e 100644 --- a/sklearn/svm/_libsvm.pxd +++ b/sklearn/svm/_libsvm.pxi @@ -1,5 +1,3 @@ -cimport numpy as np - ################################################################################ # Includes diff --git a/sklearn/svm/_libsvm.pyx b/sklearn/svm/_libsvm.pyx index 8f8e9f7465823..079a791fef3b6 100644 --- a/sklearn/svm/_libsvm.pyx +++ b/sklearn/svm/_libsvm.pyx @@ -35,6 +35,8 @@ import numpy as np cimport numpy as np from libc.stdlib cimport free +include "_libsvm.pxi" + cdef extern from *: ctypedef struct svm_parameter: pass diff --git a/sklearn/svm/src/liblinear/liblinear_helper.c b/sklearn/svm/src/liblinear/liblinear_helper.c index ffcbe86e01035..86d88e7da9273 100644 --- a/sklearn/svm/src/liblinear/liblinear_helper.c +++ b/sklearn/svm/src/liblinear/liblinear_helper.c @@ -172,7 +172,7 @@ struct problem * csr_set_problem (char *X, int double_precision_X, } -/* Create a paramater struct with and return it */ +/* Create a parameter struct with and return it */ struct parameter *set_parameter(int solver_type, double eps, double C, npy_intp nr_weight, char *weight_label, char *weight, int max_iter, unsigned seed, diff --git a/sklearn/svm/src/liblinear/linear.cpp b/sklearn/svm/src/liblinear/linear.cpp index 15202dabce7be..d9bdfb69c413d 100644 --- a/sklearn/svm/src/liblinear/linear.cpp +++ b/sklearn/svm/src/liblinear/linear.cpp @@ -515,6 +515,7 @@ Solver_MCSVM_CS::~Solver_MCSVM_CS() { delete[] B; delete[] G; + delete[] C; } int compare_double(const void *a, const void *b) diff --git a/sklearn/svm/src/libsvm/svm.cpp b/sklearn/svm/src/libsvm/svm.cpp index 8bf3aa42ed488..9321340acaaed 100644 --- a/sklearn/svm/src/libsvm/svm.cpp +++ b/sklearn/svm/src/libsvm/svm.cpp @@ -923,7 +923,7 @@ int Solver::select_working_set(int &out_i, int &out_j) // return i,j such that // i: maximizes -y_i * grad(f)_i, i in I_up(\alpha) // j: minimizes the decrease of obj value - // (if quadratic coefficeint <= 0, replace it with tau) + // (if quadratic coefficient <= 0, replace it with tau) // -y_j*grad(f)_j < -y_i*grad(f)_i, j in I_low(\alpha) double Gmax = -INF; @@ -1166,7 +1166,7 @@ int Solver_NU::select_working_set(int &out_i, int &out_j) // return i,j such that y_i = y_j and // i: maximizes -y_i * grad(f)_i, i in I_up(\alpha) // j: minimizes the decrease of obj value - // (if quadratic coefficeint <= 0, replace it with tau) + // (if quadratic coefficient <= 0, replace it with tau) // -y_j*grad(f)_j < -y_i*grad(f)_i, j in I_low(\alpha) double Gmaxp = -INF; diff --git a/sklearn/svm/src/libsvm/svm.h b/sklearn/svm/src/libsvm/svm.h index 2187e3df2916f..4002a77c93ac4 100644 --- a/sklearn/svm/src/libsvm/svm.h +++ b/sklearn/svm/src/libsvm/svm.h @@ -79,7 +79,7 @@ struct svm_model int *sv_ind; /* index of support vectors */ double *rho; /* constants in decision functions (rho[k*(k-1)/2]) */ - double *probA; /* pariwise probability information */ + double *probA; /* pairwise probability information */ double *probB; /* for classification only */ @@ -104,7 +104,7 @@ struct svm_csr_model int *sv_ind; /* index of support vectors */ double *rho; /* constants in decision functions (rho[k*(k-1)/2]) */ - double *probA; /* pariwise probability information */ + double *probA; /* pairwise probability information */ double *probB; /* for classification only */ diff --git a/sklearn/svm/tests/test_svm.py b/sklearn/svm/tests/test_svm.py index b38a4697577a3..fb811940c2971 100644 --- a/sklearn/svm/tests/test_svm.py +++ b/sklearn/svm/tests/test_svm.py @@ -20,9 +20,10 @@ from sklearn.metrics.pairwise import rbf_kernel from sklearn.utils import check_random_state from sklearn.utils._testing import assert_warns -from sklearn.utils._testing import assert_warns_message, assert_raise_message +from sklearn.utils._testing import assert_raise_message from sklearn.utils._testing import ignore_warnings from sklearn.utils._testing import assert_no_warnings +from sklearn.utils.validation import _num_samples from sklearn.utils import shuffle from sklearn.exceptions import ConvergenceWarning from sklearn.exceptions import NotFittedError, UndefinedMetricWarning @@ -125,7 +126,7 @@ def test_precomputed(): kfunc = lambda x, y: np.dot(x, y.T) clf = svm.SVC(kernel=kfunc) - clf.fit(X, Y) + clf.fit(np.array(X), Y) pred = clf.predict(T) assert_array_equal(clf.dual_coef_, [[-0.25, .25]]) @@ -542,8 +543,8 @@ def test_negative_weights_svc_leave_just_one_label(Classifier, @pytest.mark.parametrize( "Classifier, model", - [(svm.SVC, {'when-left': [0.3998, 0.4], 'when-right': [0.4, 0.3999]}), - (svm.NuSVC, {'when-left': [0.3333, 0.3333], + [(svm.SVC, {'when-left': [0.3998, 0.4], 'when-right': [0.4, 0.3999]}), + (svm.NuSVC, {'when-left': [0.3333, 0.3333], 'when-right': [0.3333, 0.3333]})], ids=['SVC', 'NuSVC'] ) @@ -638,11 +639,6 @@ def test_bad_input(): with pytest.raises(ValueError): clf.fit(X, Y) - # sample_weight bad dimensions - clf = svm.SVC() - with pytest.raises(ValueError): - clf.fit(X, Y, sample_weight=range(len(X) - 1)) - # predict with sparse input when trained with dense clf = svm.SVC().fit(X, Y) with pytest.raises(ValueError): @@ -681,19 +677,16 @@ def test_unicode_kernel(): clf.fit(X, Y) clf.predict_proba(T) _libsvm.cross_validation(iris.data, - iris.target.astype(np.float64), 5, - kernel='linear', - random_seed=0) + iris.target.astype(np.float64), 5, + kernel='linear', + random_seed=0) def test_sparse_precomputed(): clf = svm.SVC(kernel='precomputed') sparse_gram = sparse.csr_matrix([[1, 0], [0, 1]]) - try: + with pytest.raises(TypeError, match="Sparse precomputed"): clf.fit(sparse_gram, [0, 1]) - assert not "reached" - except TypeError as e: - assert "Sparse precomputed" in str(e) def test_sparse_fit_support_vectors_empty(): @@ -736,39 +729,6 @@ def test_linearsvc_parameters(): svm.LinearSVC(loss="l3").fit(X, y) -# FIXME remove in 0.23 -def test_linearsvx_loss_penalty_deprecations(): - X, y = [[0.0], [1.0]], [0, 1] - - msg = ("loss='%s' has been deprecated in favor of " - "loss='%s' as of 0.16. Backward compatibility" - " for the %s will be removed in %s") - - # LinearSVC - # loss l1 --> hinge - assert_warns_message(FutureWarning, - msg % ("l1", "hinge", "loss='l1'", "0.23"), - svm.LinearSVC(loss="l1").fit, X, y) - - # loss l2 --> squared_hinge - assert_warns_message(FutureWarning, - msg % ("l2", "squared_hinge", "loss='l2'", "0.23"), - svm.LinearSVC(loss="l2").fit, X, y) - - # LinearSVR - # loss l1 --> epsilon_insensitive - assert_warns_message(FutureWarning, - msg % ("l1", "epsilon_insensitive", "loss='l1'", - "0.23"), - svm.LinearSVR(loss="l1").fit, X, y) - - # loss l2 --> squared_epsilon_insensitive - assert_warns_message(FutureWarning, - msg % ("l2", "squared_epsilon_insensitive", - "loss='l2'", "0.23"), - svm.LinearSVR(loss="l2").fit, X, y) - - def test_linear_svx_uppercase_loss_penality_raises_error(): # Check if Upper case notation raises error at _fit_liblinear # which is called by fit @@ -1013,7 +973,7 @@ def test_svc_bad_kernel(): def test_timeout(): a = svm.SVC(kernel=lambda x, y: np.dot(x, y.T), probability=True, random_state=0, max_iter=1) - assert_warns(ConvergenceWarning, a.fit, X, Y) + assert_warns(ConvergenceWarning, a.fit, np.array(X), Y) def test_unfitted(): @@ -1059,8 +1019,9 @@ def test_svr_coef_sign(): for svr in [svm.SVR(kernel='linear'), svm.NuSVR(kernel='linear'), svm.LinearSVR()]: svr.fit(X, y) - assert_array_almost_equal(svr.predict(X), - np.dot(X, svr.coef_.ravel()) + svr.intercept_) + assert_array_almost_equal( + svr.predict(X), np.dot(X, svr.coef_.ravel()) + svr.intercept_ + ) def test_linear_svc_intercept_scaling(): @@ -1127,7 +1088,7 @@ def test_ovr_decision_function(): base_points * [-1, 1], # Q2 base_points * [-1, -1], # Q3 base_points * [1, -1] # Q4 - )) + )) y_test = [0] * 2 + [1] * 2 + [2] * 2 + [3] * 2 @@ -1266,3 +1227,58 @@ def test_n_support_oneclass_svr(): assert reg.n_support_ == reg.support_vectors_.shape[0] assert reg.n_support_.size == 1 assert reg.n_support_ == 4 + + +# TODO: Remove in 0.25 when probA_ and probB_ are deprecated +@pytest.mark.parametrize("SVMClass, data", [ + (svm.OneClassSVM, (X, )), + (svm.SVR, (X, Y)) +]) +@pytest.mark.parametrize("deprecated_prob", ["probA_", "probB_"]) +def test_svm_probA_proB_deprecated(SVMClass, data, deprecated_prob): + clf = SVMClass().fit(*data) + + msg = ("The {} attribute is deprecated in version 0.23 and will be " + "removed in version 0.25.").format(deprecated_prob) + with pytest.warns(FutureWarning, match=msg): + getattr(clf, deprecated_prob) + + +@pytest.mark.parametrize("Estimator", [svm.SVC, svm.SVR]) +def test_custom_kernel_not_array_input(Estimator): + """Test using a custom kernel that is not fed with array-like for floats""" + data = ["A A", "A", "B", "B B", "A B"] + X = np.array([[2, 0], [1, 0], [0, 1], [0, 2], [1, 1]]) # count encoding + y = np.array([1, 1, 2, 2, 1]) + + def string_kernel(X1, X2): + assert isinstance(X1[0], str) + n_samples1 = _num_samples(X1) + n_samples2 = _num_samples(X2) + K = np.zeros((n_samples1, n_samples2)) + for ii in range(n_samples1): + for jj in range(ii, n_samples2): + K[ii, jj] = X1[ii].count('A') * X2[jj].count('A') + K[ii, jj] += X1[ii].count('B') * X2[jj].count('B') + K[jj, ii] = K[ii, jj] + return K + + K = string_kernel(data, data) + assert_array_equal(np.dot(X, X.T), K) + + svc1 = Estimator(kernel=string_kernel).fit(data, y) + svc2 = Estimator(kernel='linear').fit(X, y) + svc3 = Estimator(kernel='precomputed').fit(K, y) + + assert svc1.score(data, y) == svc3.score(K, y) + assert svc1.score(data, y) == svc2.score(X, y) + if hasattr(svc1, 'decision_function'): # classifier + assert_allclose(svc1.decision_function(data), + svc2.decision_function(X)) + assert_allclose(svc1.decision_function(data), + svc3.decision_function(K)) + assert_array_equal(svc1.predict(data), svc2.predict(X)) + assert_array_equal(svc1.predict(data), svc3.predict(K)) + else: # regressor + assert_allclose(svc1.predict(data), svc2.predict(X)) + assert_allclose(svc1.predict(data), svc3.predict(K)) diff --git a/sklearn/tests/test_base.py b/sklearn/tests/test_base.py index 155dbcaaa1f6c..95f7b01f27058 100644 --- a/sklearn/tests/test_base.py +++ b/sklearn/tests/test_base.py @@ -196,6 +196,14 @@ def test_clone_estimator_types(): assert clf.empty is clf2.empty +def test_clone_class_rather_than_instance(): + # Check that clone raises expected error message when + # cloning class rather than instance + msg = "You should provide an instance of scikit-learn estimator" + with pytest.raises(TypeError, match=msg): + clone(MyEstimator) + + def test_repr(): # Smoke test the repr of the base estimator. my_estimator = MyEstimator() @@ -490,29 +498,6 @@ def test_tag_inheritance(): assert inherit_diamond_tag_est._get_tags()['allow_nan'] -# XXX: Remove in 0.23 -def test_regressormixin_score_multioutput(): - from sklearn.linear_model import LinearRegression - # no warnings when y_type is continuous - X = [[1], [2], [3]] - y = [1, 2, 3] - reg = LinearRegression().fit(X, y) - assert_no_warnings(reg.score, X, y) - # warn when y_type is continuous-multioutput - y = [[1, 2], [2, 3], [3, 4]] - reg = LinearRegression().fit(X, y) - msg = ("The default value of multioutput (not exposed in " - "score method) will change from 'variance_weighted' " - "to 'uniform_average' in 0.23 to keep consistent " - "with 'metrics.r2_score'. To specify the default " - "value manually and avoid the warning, please " - "either call 'metrics.r2_score' directly or make a " - "custom scorer with 'metrics.make_scorer' (the " - "built-in scorer 'r2' uses " - "multioutput='uniform_average').") - assert_warns_message(FutureWarning, msg, reg.score, X, y) - - def test_warns_on_get_params_non_attribute(): class MyEstimator(BaseEstimator): def __init__(self, param=5): diff --git a/sklearn/tests/test_common.py b/sklearn/tests/test_common.py index 221dd52834c90..d769bb630bd03 100644 --- a/sklearn/tests/test_common.py +++ b/sklearn/tests/test_common.py @@ -20,10 +20,10 @@ from sklearn.utils import all_estimators from sklearn.utils._testing import ignore_warnings from sklearn.exceptions import ConvergenceWarning -from sklearn.utils.estimator_checks import check_estimator +from sklearn.utils.estimator_checks import check_estimator, _safe_tags import sklearn -from sklearn.base import RegressorMixin, BiclusterMixin +from sklearn.base import BiclusterMixin from sklearn.linear_model._base import LinearClassifierMixin from sklearn.linear_model import LogisticRegression @@ -78,8 +78,6 @@ def _tested_estimators(): for name, Estimator in all_estimators(): if issubclass(Estimator, BiclusterMixin): continue - if name.startswith("_"): - continue try: estimator = _construct_instance(Estimator) except SkipTest: @@ -89,7 +87,7 @@ def _tested_estimators(): @parametrize_with_checks(_tested_estimators()) -def test_estimators(estimator, check): +def test_estimators(estimator, check, request): # Common tests for estimator instances with ignore_warnings(category=(FutureWarning, ConvergenceWarning, @@ -133,7 +131,8 @@ def test_configure(): setup_path = os.path.abspath(os.path.join(sklearn.__path__[0], '..')) setup_filename = os.path.join(setup_path, 'setup.py') if not os.path.exists(setup_filename): - return + pytest.skip('setup.py not available') + # XXX unreached code as of v0.22 try: os.chdir(setup_path) old_argv = sys.argv @@ -181,15 +180,13 @@ def test_import_all_consistency(): for modname in submods + ['sklearn']: if ".tests." in modname: continue - if IS_PYPY and ('_svmlight_format' in modname or + if IS_PYPY and ('_svmlight_format_io' in modname or 'feature_extraction._hashing_fast' in modname): continue package = __import__(modname, fromlist="dummy") for name in getattr(package, '__all__', ()): - if getattr(package, name, None) is None: - raise AttributeError( - "Module '{0}' has no attribute '{1}'".format( - modname, name)) + assert hasattr(package, name),\ + "Module '{0}' has no attribute '{1}'".format(modname, name) def test_root_import_all_completeness(): diff --git a/sklearn/tests/test_discriminant_analysis.py b/sklearn/tests/test_discriminant_analysis.py index 7b3e94bea793c..dcd4009a47a2d 100644 --- a/sklearn/tests/test_discriminant_analysis.py +++ b/sklearn/tests/test_discriminant_analysis.py @@ -332,7 +332,6 @@ def test_lda_store_covariance(): @pytest.mark.parametrize('n_features', [3, 5]) @pytest.mark.parametrize('n_classes', [5, 3]) def test_lda_dimension_warning(n_classes, n_features): - # FIXME: Future warning to be removed in 0.23 rng = check_random_state(0) n_samples = 10 X = rng.randn(n_samples, n_features) @@ -348,22 +347,14 @@ def test_lda_dimension_warning(n_classes, n_features): for n_components in [max_components + 1, max(n_features, n_classes - 1) + 1]: - # if n_components > min(n_classes - 1, n_features), raise warning + # if n_components > min(n_classes - 1, n_features), raise error. # We test one unit higher than max_components, and then something # larger than both n_features and n_classes - 1 to ensure the test # works for any value of n_component lda = LinearDiscriminantAnalysis(n_components=n_components) - msg = ("n_components cannot be larger than min(n_features, " - "n_classes - 1). Using min(n_features, " - "n_classes - 1) = min(%d, %d - 1) = %d components." % - (n_features, n_classes, max_components)) - assert_warns_message(ChangedBehaviorWarning, msg, lda.fit, X, y) - future_msg = ("In version 0.23, setting n_components > min(" - "n_features, n_classes - 1) will raise a " - "ValueError. You should set n_components to None" - " (default), or a value smaller or equal to " - "min(n_features, n_classes - 1).") - assert_warns_message(FutureWarning, future_msg, lda.fit, X, y) + msg = "n_components cannot be larger than " + with pytest.raises(ValueError, match=msg): + lda.fit(X, y) @pytest.mark.parametrize("data_type, expected_type", [ diff --git a/sklearn/tests/test_docstring_parameters.py b/sklearn/tests/test_docstring_parameters.py index 31c7d268737b9..55af69ca6c10e 100644 --- a/sklearn/tests/test_docstring_parameters.py +++ b/sklearn/tests/test_docstring_parameters.py @@ -9,14 +9,20 @@ from pkgutil import walk_packages from inspect import signature +import numpy as np + import sklearn from sklearn.utils import IS_PYPY -from sklearn.utils._testing import SkipTest from sklearn.utils._testing import check_docstring_parameters from sklearn.utils._testing import _get_func_name from sklearn.utils._testing import ignore_warnings +from sklearn.utils._testing import all_estimators +from sklearn.utils.estimator_checks import _safe_tags +from sklearn.utils.estimator_checks import _enforce_estimator_tags_y +from sklearn.utils.estimator_checks import _enforce_estimator_tags_x from sklearn.utils.deprecation import _is_deprecated from sklearn.externals._pep562 import Pep562 +from sklearn.datasets import make_classification import pytest @@ -60,11 +66,10 @@ def test_docstring_parameters(): # Test module docstring formatting # Skip test if numpydoc is not found - try: - import numpydoc # noqa - except ImportError: - raise SkipTest("numpydoc is required to test the docstrings") + pytest.importorskip('numpydoc', + reason="numpydoc is required to test the docstrings") + # XXX unreached code as of v0.22 from numpydoc import docscrape incorrect = [] @@ -140,7 +145,7 @@ def test_tabs(): for importer, modname, ispkg in walk_packages(sklearn.__path__, prefix='sklearn.'): - if IS_PYPY and ('_svmlight_format' in modname or + if IS_PYPY and ('_svmlight_format_io' in modname or 'feature_extraction._hashing_fast' in modname): continue @@ -158,5 +163,91 @@ def test_tabs(): except IOError: # user probably should have run "make clean" continue assert '\t' not in source, ('"%s" has tabs, please remove them ', - 'or add it to theignore list' + 'or add it to the ignore list' % modname) + + +@pytest.mark.parametrize('name, Estimator', + all_estimators()) +def test_fit_docstring_attributes(name, Estimator): + pytest.importorskip('numpydoc') + from numpydoc import docscrape + + doc = docscrape.ClassDoc(Estimator) + attributes = doc['Attributes'] + + IGNORED = {'ClassifierChain', 'ColumnTransformer', 'CountVectorizer', + 'DictVectorizer', 'FeatureUnion', 'GaussianRandomProjection', + 'GridSearchCV', 'MultiOutputClassifier', 'MultiOutputRegressor', + 'NoSampleWeightWrapper', 'OneVsOneClassifier', + 'OneVsRestClassifier', 'OutputCodeClassifier', 'Pipeline', + 'RFE', 'RFECV', 'RandomizedSearchCV', 'RegressorChain', + 'SelectFromModel', 'SparseCoder', 'SparseRandomProjection', + 'SpectralBiclustering', 'StackingClassifier', + 'StackingRegressor', 'TfidfVectorizer', 'VotingClassifier', + 'VotingRegressor'} + if Estimator.__name__ in IGNORED or Estimator.__name__.startswith('_'): + pytest.skip("Estimator cannot be fit easily to test fit attributes") + + est = Estimator() + + if Estimator.__name__ == 'SelectKBest': + est.k = 2 + + if Estimator.__name__ == 'DummyClassifier': + est.strategy = "stratified" + + X, y = make_classification(n_samples=20, n_features=3, + n_redundant=0, n_classes=2, + random_state=2) + + y = _enforce_estimator_tags_y(est, y) + X = _enforce_estimator_tags_x(est, X) + + if '1dlabels' in _safe_tags(est, 'X_types'): + est.fit(y) + elif '2dlabels' in _safe_tags(est, 'X_types'): + est.fit(np.c_[y, y]) + else: + est.fit(X, y) + + skipped_attributes = {'n_features_in_'} + + for attr in attributes: + if attr.name in skipped_attributes: + continue + desc = ' '.join(attr.desc).lower() + # As certain attributes are present "only" if a certain parameter is + # provided, this checks if the word "only" is present in the attribute + # description, and if not the attribute is required to be present. + if 'only ' not in desc: + assert hasattr(est, attr.name) + + IGNORED = {'BayesianRidge', 'Birch', 'CCA', 'CategoricalNB', 'ElasticNet', + 'ElasticNetCV', 'GaussianProcessClassifier', + 'GradientBoostingRegressor', 'HistGradientBoostingClassifier', + 'HistGradientBoostingRegressor', 'IsolationForest', + 'KNeighborsClassifier', 'KNeighborsRegressor', + 'KNeighborsTransformer', 'KernelCenterer', 'KernelDensity', + 'LarsCV', 'Lasso', 'LassoLarsCV', 'LassoLarsIC', + 'LatentDirichletAllocation', 'LocalOutlierFactor', 'MDS', + 'MiniBatchKMeans', 'MLPClassifier', 'MLPRegressor', + 'MultiTaskElasticNet', 'MultiTaskElasticNetCV', + 'MultiTaskLasso', 'MultiTaskLassoCV', 'NearestNeighbors', + 'NuSVR', 'OAS', 'OneClassSVM', 'OrthogonalMatchingPursuit', + 'PLSCanonical', 'PLSRegression', 'PLSSVD', + 'PassiveAggressiveClassifier', 'Perceptron', 'RBFSampler', + 'RadiusNeighborsClassifier', 'RadiusNeighborsRegressor', + 'RadiusNeighborsTransformer', 'RandomTreesEmbedding', 'SVR', + 'SkewedChi2Sampler'} + if Estimator.__name__ in IGNORED: + pytest.xfail( + reason="Classifier has too many undocumented attributes.") + + fit_attr = [k for k in est.__dict__.keys() if k.endswith('_') + and not k.startswith('_')] + fit_attr_names = [attr.name for attr in attributes] + undocumented_attrs = set(fit_attr).difference(fit_attr_names) + undocumented_attrs = set(undocumented_attrs).difference(skipped_attributes) + assert not undocumented_attrs,\ + "Undocumented attributes: {}".format(undocumented_attrs) diff --git a/sklearn/tests/test_dummy.py b/sklearn/tests/test_dummy.py index 55f3abc77b0de..38abb0b158fd3 100644 --- a/sklearn/tests/test_dummy.py +++ b/sklearn/tests/test_dummy.py @@ -708,7 +708,6 @@ def test_dummy_regressor_return_std(): assert_array_equal(y_pred_list[1], y_std_expected) -@pytest.mark.filterwarnings('ignore: The default value of multioutput') # 0.23 @pytest.mark.parametrize("y,y_test", [ ([1, 1, 1, 2], [1.25] * 4), (np.array([[2, 2], @@ -757,6 +756,17 @@ def test_dtype_of_classifier_probas(strategy): assert probas.dtype == np.float64 +@pytest.mark.filterwarnings("ignore:The default value of strategy.*") # 0.24 +@pytest.mark.parametrize('Dummy', (DummyRegressor, DummyClassifier)) +def test_n_features_in_(Dummy): + X = [[1, 2]] + y = [0] + d = Dummy() + assert not hasattr(d, 'n_features_in_') + d.fit(X, y) + assert d.n_features_in_ is None + + @pytest.mark.parametrize("Dummy", (DummyRegressor, DummyClassifier)) def test_outputs_2d_deprecation(Dummy): X = [[1, 2]] diff --git a/sklearn/tests/test_import_deprecations.py b/sklearn/tests/test_import_deprecations.py index 13b31e89b2862..29c4259fe1e5a 100644 --- a/sklearn/tests/test_import_deprecations.py +++ b/sklearn/tests/test_import_deprecations.py @@ -24,6 +24,12 @@ def test_import_is_deprecated(deprecated_path, importee): # TODO: remove in 0.24 + # Special case for: + # https://github.com/scikit-learn/scikit-learn/issues/15842 + if deprecated_path in ("sklearn.decomposition.dict_learning", + "sklearn.inspection.partial_dependence"): + pytest.skip("No warning can be raised for " + deprecated_path) + expected_message = ( "The {deprecated_path} module is deprecated in version " "0.22 and will be removed in version 0.24. " diff --git a/sklearn/tests/test_kernel_approximation.py b/sklearn/tests/test_kernel_approximation.py index dcdc5dab2a72b..8d37ce218f227 100644 --- a/sklearn/tests/test_kernel_approximation.py +++ b/sklearn/tests/test_kernel_approximation.py @@ -20,6 +20,10 @@ Y /= Y.sum(axis=1)[:, np.newaxis] +def _linear_kernel(X, Y): + return np.dot(X, Y.T) + + def test_additive_chi2_sampler(): # test that AdditiveChi2Sampler approximates kernel on random data @@ -118,6 +122,18 @@ def test_skewed_chi2_sampler(): assert_raises(ValueError, transform.transform, Y_neg) +def test_additive_chi2_sampler_exceptions(): + """Ensures correct error message""" + transformer = AdditiveChi2Sampler() + X_neg = X.copy() + X_neg[0, 0] = -1 + with pytest.raises(ValueError, match="X in AdditiveChi2Sampler.fit"): + transformer.fit(X_neg) + with pytest.raises(ValueError, match="X in AdditiveChi2Sampler.transform"): + transformer.fit(X) + transformer.transform(X_neg) + + def test_rbf_sampler(): # test that RBFSampler approximates kernel on random data # compute exact kernel @@ -164,9 +180,7 @@ def test_nystroem_approximation(): assert X_transformed.shape == (X.shape[0], 2) # test callable kernel - def linear_kernel(X, Y): - return np.dot(X, Y.T) - trans = Nystroem(n_components=2, kernel=linear_kernel, random_state=rnd) + trans = Nystroem(n_components=2, kernel=_linear_kernel, random_state=rnd) X_transformed = trans.fit(X).transform(X) assert X_transformed.shape == (X.shape[0], 2) @@ -244,14 +258,11 @@ def logging_histogram_kernel(x, y, log): kernel_params={'log': kernel_log}).fit(X) assert len(kernel_log) == n_samples * (n_samples - 1) / 2 - def linear_kernel(X, Y): - return np.dot(X, Y.T) - # if degree, gamma or coef0 is passed, we raise a warning msg = "Don't pass gamma, coef0 or degree to Nystroem" params = ({'gamma': 1}, {'coef0': 1}, {'degree': 2}) for param in params: - ny = Nystroem(kernel=linear_kernel, **param) + ny = Nystroem(kernel=_linear_kernel, **param) with pytest.raises(ValueError, match=msg): ny.fit(X) diff --git a/sklearn/tests/test_multiclass.py b/sklearn/tests/test_multiclass.py index ef0aa888f2ab9..33eb5da939725 100644 --- a/sklearn/tests/test_multiclass.py +++ b/sklearn/tests/test_multiclass.py @@ -76,8 +76,6 @@ def test_ovr_fit_predict(): assert np.mean(iris.target == pred) > 0.65 -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_ovr_partial_fit(): # Test if partial_fit is working as intended X, y = shuffle(iris.data, iris.target, random_state=0) @@ -602,8 +600,6 @@ def test_ovo_gridsearch(): assert best_C in Cs -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_ovo_ties(): # Test that ties are broken using the decision function, # not defaulting to the smallest label @@ -629,8 +625,6 @@ def test_ovo_ties(): assert ovo_prediction[0] == normalized_confidences[0].argmax() -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_ovo_ties2(): # test that ties can not only be won by the first two labels X = np.array([[1, 2], [2, 1], [-2, 1], [-2, -1]]) diff --git a/sklearn/tests/test_multioutput.py b/sklearn/tests/test_multioutput.py index cd87ad3fc863d..d312231d0430a 100644 --- a/sklearn/tests/test_multioutput.py +++ b/sklearn/tests/test_multioutput.py @@ -17,6 +17,7 @@ from sklearn.exceptions import NotFittedError from sklearn.linear_model import Lasso from sklearn.linear_model import LogisticRegression +from sklearn.linear_model import OrthogonalMatchingPursuit from sklearn.linear_model import Ridge from sklearn.linear_model import SGDClassifier from sklearn.linear_model import SGDRegressor @@ -30,6 +31,7 @@ from sklearn.base import ClassifierMixin from sklearn.utils import shuffle from sklearn.model_selection import GridSearchCV +from sklearn.dummy import DummyRegressor, DummyClassifier def test_multi_target_regression(): @@ -50,8 +52,6 @@ def test_multi_target_regression(): assert_almost_equal(references, y_pred) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_multi_target_regression_partial_fit(): X, y = datasets.make_regression(n_targets=3) X_train, y_train = X[:50], y[:50] @@ -104,7 +104,7 @@ def test_multi_target_sample_weights_api(): y = [[3.141, 2.718], [2.718, 3.141]] w = [0.8, 0.6] - rgr = MultiOutputRegressor(Lasso()) + rgr = MultiOutputRegressor(OrthogonalMatchingPursuit()) assert_raises_regex(ValueError, "does not support sample weights", rgr.fit, X, y, w) @@ -113,8 +113,6 @@ def test_multi_target_sample_weights_api(): rgr.fit(X, y, w) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_multi_target_sample_weight_partial_fit(): # weighted regressor X = [[1, 2, 3], [4, 5, 6]] @@ -219,8 +217,6 @@ def custom_scorer(estimator, X, y): multi_target_linear.predict_proba(X) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_multi_output_classification_partial_fit(): # test if multi_target initializes correctly with base estimator and fit # assert predictions work as expected for predict @@ -252,8 +248,6 @@ def test_multi_output_classification_partial_fit(): assert_array_equal(sgd_linear_clf.predict(X), second_predictions[:, i]) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_multi_output_classification_partial_fit_no_first_classes_exception(): sgd_linear_clf = SGDClassifier(loss='log', random_state=1, max_iter=5) multi_target_linear = MultiOutputClassifier(sgd_linear_clf) @@ -368,8 +362,6 @@ def test_multi_output_classification_sample_weights(): assert_almost_equal(clf.predict(X_test), clf_w.predict(X_test)) -# 0.23. warning about tol not having its correct default value. -@pytest.mark.filterwarnings('ignore:max_iter and tol parameters have been') def test_multi_output_classification_partial_fit_sample_weights(): # weighted classifier Xw = [[1, 2, 3], [4, 5, 6], [1.5, 2.5, 3.5]] @@ -571,3 +563,51 @@ class A(MultiOutputEstimator, MultiOutputRegressor): with pytest.warns(FutureWarning, match="is deprecated in version 0.22"): A(SGDRegressor(random_state=0, max_iter=5)) + + +class DummyRegressorWithFitParams(DummyRegressor): + def fit(self, X, y, sample_weight=None, **fit_params): + self._fit_params = fit_params + return super().fit(X, y, sample_weight) + + +class DummyClassifierWithFitParams(DummyClassifier): + def fit(self, X, y, sample_weight=None, **fit_params): + self._fit_params = fit_params + return super().fit(X, y, sample_weight) + + +@pytest.mark.parametrize( + "estimator, dataset", + [(MultiOutputClassifier(DummyClassifierWithFitParams(strategy="prior")), + datasets.make_multilabel_classification()), + (MultiOutputRegressor(DummyRegressorWithFitParams()), + datasets.make_regression(n_targets=3))]) +def test_multioutput_estimator_with_fit_params(estimator, dataset): + X, y = dataset + some_param = np.zeros_like(X) + estimator.fit(X, y, some_param=some_param) + for dummy_estimator in estimator.estimators_: + assert 'some_param' in dummy_estimator._fit_params + + +def test_regressor_chain_w_fit_params(): + # Make sure fit_params are properly propagated to the sub-estimators + rng = np.random.RandomState(0) + X, y = datasets.make_regression(n_targets=3) + weight = rng.rand(y.shape[0]) + + class MySGD(SGDRegressor): + + def fit(self, X, y, **fit_params): + self.sample_weight_ = fit_params['sample_weight'] + super().fit(X, y, **fit_params) + + model = RegressorChain(MySGD()) + + # Fitting with params + fit_param = {'sample_weight': weight} + model.fit(X, y, **fit_param) + + for est in model.estimators_: + assert est.sample_weight_ is weight diff --git a/sklearn/tests/test_naive_bayes.py b/sklearn/tests/test_naive_bayes.py index 1c00438eb8ab9..1f0f9347a188c 100644 --- a/sklearn/tests/test_naive_bayes.py +++ b/sklearn/tests/test_naive_bayes.py @@ -663,10 +663,15 @@ def test_categoricalnb(): # Check error is raised for X with negative entries X = np.array([[0, -1]]) y = np.array([1]) - error_msg = "X must not contain negative values." + error_msg = "Negative values in data passed to CategoricalNB (input X)" assert_raise_message(ValueError, error_msg, clf.predict, X) assert_raise_message(ValueError, error_msg, clf.fit, X, y) + # Check error is raised for incorrect X + X = np.array([[1, 4, 1], [2, 5, 6]]) + msg = "Expected input with 2 features, got 3 instead" + assert_raise_message(ValueError, msg, clf.predict, X) + # Test alpha X3_test = np.array([[2, 5]]) # alpha=1 increases the count of all categories by one so the final diff --git a/sklearn/tests/test_pipeline.py b/sklearn/tests/test_pipeline.py index bd9246269f0f4..b9c2e26abac61 100644 --- a/sklearn/tests/test_pipeline.py +++ b/sklearn/tests/test_pipeline.py @@ -34,6 +34,8 @@ from sklearn.datasets import load_iris from sklearn.preprocessing import StandardScaler from sklearn.feature_extraction.text import CountVectorizer +from sklearn.experimental import enable_hist_gradient_boosting # noqa +from sklearn.ensemble import HistGradientBoostingClassifier iris = load_iris() @@ -1161,6 +1163,49 @@ def test_verbose(est, method, pattern, capsys): assert re.match(pattern, capsys.readouterr().out) +def test_n_features_in_pipeline(): + # make sure pipelines delegate n_features_in to the first step + + X = [[1, 2], [3, 4], [5, 6]] + y = [0, 1, 2] + + ss = StandardScaler() + gbdt = HistGradientBoostingClassifier() + pipe = make_pipeline(ss, gbdt) + assert not hasattr(pipe, 'n_features_in_') + pipe.fit(X, y) + assert pipe.n_features_in_ == ss.n_features_in_ == 2 + + # if the first step has the n_features_in attribute then the pipeline also + # has it, even though it isn't fitted. + ss = StandardScaler() + gbdt = HistGradientBoostingClassifier() + pipe = make_pipeline(ss, gbdt) + ss.fit(X, y) + assert pipe.n_features_in_ == ss.n_features_in_ == 2 + assert not hasattr(gbdt, 'n_features_in_') + + +def test_n_features_in_feature_union(): + # make sure FeatureUnion delegates n_features_in to the first transformer + + X = [[1, 2], [3, 4], [5, 6]] + y = [0, 1, 2] + + ss = StandardScaler() + fu = make_union(ss) + assert not hasattr(fu, 'n_features_in_') + fu.fit(X, y) + assert fu.n_features_in_ == ss.n_features_in_ == 2 + + # if the first step has the n_features_in attribute then the feature_union + # also has it, even though it isn't fitted. + ss = StandardScaler() + fu = make_union(ss) + ss.fit(X, y) + assert fu.n_features_in_ == ss.n_features_in_ == 2 + + def test_feature_union_fit_params(): # Regression test for issue: #15117 class Dummy(TransformerMixin, BaseEstimator): diff --git a/sklearn/tree/_classes.py b/sklearn/tree/_classes.py index e0ccb723d2e36..09481aefeed41 100644 --- a/sklearn/tree/_classes.py +++ b/sklearn/tree/_classes.py @@ -104,8 +104,8 @@ def __init__(self, self.min_samples_leaf = min_samples_leaf self.min_weight_fraction_leaf = min_weight_fraction_leaf self.max_features = max_features - self.random_state = random_state self.max_leaf_nodes = max_leaf_nodes + self.random_state = random_state self.min_impurity_decrease = min_impurity_decrease self.min_impurity_split = min_impurity_split self.class_weight = class_weight @@ -146,7 +146,7 @@ def fit(self, X, y, sample_weight=None, check_input=True, raise ValueError("ccp_alpha must be greater than or equal to 0") if check_input: - X = check_array(X, dtype=DTYPE, accept_sparse="csc") + X = self._validate_data(X, dtype=DTYPE, accept_sparse="csc") y = check_array(y, ensure_2d=False, dtype=None) if issparse(X): X.sort_indices() @@ -197,7 +197,7 @@ def fit(self, X, y, sample_weight=None, check_input=True, y = np.ascontiguousarray(y, dtype=DOUBLE) # Check parameters - max_depth = ((2 ** 31) - 1 if self.max_depth is None + max_depth = (np.iinfo(np.int32).max if self.max_depth is None else self.max_depth) max_leaf_nodes = (-1 if self.max_leaf_nodes is None else self.max_leaf_nodes) @@ -244,9 +244,9 @@ def fit(self, X, y, sample_weight=None, check_input=True, elif self.max_features == "log2": max_features = max(1, int(np.log2(self.n_features_))) else: - raise ValueError( - 'Invalid value for max_features. Allowed string ' - 'values are "auto", "sqrt" or "log2".') + raise ValueError("Invalid value for max_features. " + "Allowed string values are 'auto', " + "'sqrt' or 'log2'.") elif self.max_features is None: max_features = self.n_features_ elif isinstance(self.max_features, numbers.Integral): @@ -293,19 +293,19 @@ def fit(self, X, y, sample_weight=None, check_input=True, min_weight_leaf = (self.min_weight_fraction_leaf * np.sum(sample_weight)) - if self.min_impurity_split is not None: + min_impurity_split = self.min_impurity_split + if min_impurity_split is not None: warnings.warn("The min_impurity_split parameter is deprecated. " - "Its default value will change from 1e-7 to 0 in " + "Its default value has changed from 1e-7 to 0 in " "version 0.23, and it will be removed in 0.25. " "Use the min_impurity_decrease parameter instead.", FutureWarning) - min_impurity_split = self.min_impurity_split - else: - min_impurity_split = 1e-7 - if min_impurity_split < 0.: - raise ValueError("min_impurity_split must be greater than " - "or equal to 0") + if min_impurity_split < 0.: + raise ValueError("min_impurity_split must be greater than " + "or equal to 0") + else: + min_impurity_split = 0 if self.min_impurity_decrease < 0.: raise ValueError("min_impurity_decrease must be greater than " @@ -401,12 +401,12 @@ def predict(self, X, check_input=True): Parameters ---------- - X : array-like or sparse matrix of shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. - check_input : bool, (default=True) + check_input : bool, default=True Allow to bypass several input checking. Don't use this parameter unless you know what you do. @@ -445,8 +445,7 @@ def predict(self, X, check_input=True): return proba[:, :, 0] def apply(self, X, check_input=True): - """ - Return the index of the leaf that each sample is predicted as. + """Return the index of the leaf that each sample is predicted as. .. versionadded:: 0.17 @@ -457,13 +456,13 @@ def apply(self, X, check_input=True): ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. - check_input : bool, (default=True) + check_input : bool, default=True Allow to bypass several input checking. Don't use this parameter unless you know what you do. Returns ------- - X_leaves : array_like, shape = [n_samples,] + X_leaves : array-like of shape (n_samples,) For each datapoint x in X, return the index of the leaf x ends up in. Leaves are numbered within ``[0; self.tree_.node_count)``, possibly with gaps in the @@ -485,14 +484,14 @@ def decision_path(self, X, check_input=True): ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. - check_input : bool, (default=True) + check_input : bool, default=True Allow to bypass several input checking. Don't use this parameter unless you know what you do. Returns ------- - indicator : sparse csr array, shape = [n_samples, n_nodes] - Return a node indicator matrix where non zero elements + indicator : sparse matrix of shape (n_samples, n_nodes) + Return a node indicator CSR matrix where non zero elements indicates that the samples goes through the nodes. """ X = self._validate_X_predict(X, check_input) @@ -546,8 +545,8 @@ def cost_complexity_pruning_path(self, X, y, sample_weight=None): Returns ------- - ccp_path : Bunch - Dictionary-like object, with attributes: + ccp_path : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. ccp_alphas : ndarray Effective alphas of subtree during pruning. @@ -568,10 +567,15 @@ def feature_importances_(self): reduction of the criterion brought by that feature. It is also known as the Gini importance. + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + Returns ------- - feature_importances_ : array, shape = [n_features] - Normalized total reduction of critera by feature (Gini importance). + feature_importances_ : ndarray of shape (n_features,) + Normalized total reduction of criteria by feature + (Gini importance). """ check_is_fitted(self) @@ -589,21 +593,21 @@ class DecisionTreeClassifier(ClassifierMixin, BaseDecisionTree): Parameters ---------- - criterion : str, optional (default="gini") + criterion : {"gini", "entropy"}, default="gini" The function to measure the quality of a split. Supported criteria are "gini" for the Gini impurity and "entropy" for the information gain. - splitter : str, optional (default="best") + splitter : {"best", "random"}, default="best" The strategy used to choose the split at each node. Supported strategies are "best" to choose the best split and "random" to choose the best random split. - max_depth : int or None, optional (default=None) + max_depth : int, default=None The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples. - min_samples_split : int, float, optional (default=2) + min_samples_split : int or float, default=2 The minimum number of samples required to split an internal node: - If int, then consider `min_samples_split` as the minimum number. @@ -614,7 +618,7 @@ class DecisionTreeClassifier(ClassifierMixin, BaseDecisionTree): .. versionchanged:: 0.18 Added float values for fractions. - min_samples_leaf : int, float, optional (default=1) + min_samples_leaf : int or float, default=1 The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least ``min_samples_leaf`` training samples in each of the left and @@ -629,12 +633,12 @@ class DecisionTreeClassifier(ClassifierMixin, BaseDecisionTree): .. versionchanged:: 0.18 Added float values for fractions. - min_weight_fraction_leaf : float, optional (default=0.) + min_weight_fraction_leaf : float, default=0.0 The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. - max_features : int, float, str or None, optional (default=None) + max_features : int, float or {"auto", "sqrt", "log2"}, default=None The number of features to consider when looking for the best split: - If int, then consider `max_features` features at each split. @@ -650,18 +654,24 @@ class DecisionTreeClassifier(ClassifierMixin, BaseDecisionTree): valid partition of the node samples is found, even if it requires to effectively inspect more than ``max_features`` features. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - max_leaf_nodes : int or None, optional (default=None) + random_state : int, RandomState instance, default=None + Controls the randomness of the estimator. The features are always + randomly permuted at each split, even if ``splitter`` is set to + ``"best"``. When ``max_features < n_features``, the algorithm will + select ``max_features`` at random at each split before finding the best + split among them. But the best found split may vary across different + runs, even if ``max_features=n_features``. That is the case, if the + improvement of the criterion is identical for several splits and one + split has to be selected at random. To obtain a deterministic behaviour + during fitting, ``random_state`` has to be fixed to an integer. + See :term:`Glossary ` for details. + + max_leaf_nodes : int, default=None Grow a tree with ``max_leaf_nodes`` in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. - min_impurity_decrease : float, optional (default=0.) + min_impurity_decrease : float, default=0.0 A node will be split if this split induces a decrease of the impurity greater than or equal to this value. @@ -679,19 +689,19 @@ class DecisionTreeClassifier(ClassifierMixin, BaseDecisionTree): .. versionadded:: 0.19 - min_impurity_split : float, default=1e-7 + min_impurity_split : float, default=0 Threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf. .. deprecated:: 0.19 ``min_impurity_split`` has been deprecated in favor of ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` will change from 1e-7 to 0 in 0.23 and it + ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it will be removed in 0.25. Use ``min_impurity_decrease`` instead. - class_weight : dict, list of dicts, "balanced" or None, default=None + class_weight : dict, list of dict or "balanced", default=None Weights associated with classes in the form ``{class_label: weight}``. - If not given, all classes are supposed to have weight one. For + If None, all classes are supposed to have weight one. For multi-output problems, a list of dicts can be provided in the same order as the columns of y. @@ -715,7 +725,7 @@ class DecisionTreeClassifier(ClassifierMixin, BaseDecisionTree): .. deprecated:: 0.22 - ccp_alpha : non-negative float, optional (default=0.0) + ccp_alpha : non-negative float, default=0.0 Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ``ccp_alpha`` will be chosen. By default, no pruning is performed. See @@ -725,20 +735,25 @@ class DecisionTreeClassifier(ClassifierMixin, BaseDecisionTree): Attributes ---------- - classes_ : array of shape (n_classes,) or a list of such arrays + classes_ : ndarray of shape (n_classes,) or list of ndarray The classes labels (single output problem), or a list of arrays of class labels (multi-output problem). feature_importances_ : ndarray of shape (n_features,) - The feature importances. The higher, the more important the - feature. The importance of a feature is computed as the (normalized) + The impurity-based feature importances. + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) total reduction of the criterion brought by that feature. It is also known as the Gini importance [4]_. - max_features_ : int, + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + + max_features_ : int The inferred value of max_features. - n_classes_ : int or list + n_classes_ : int or list of int The number of classes (for single output problems), or a list containing the number of classes for each output (for multi-output problems). @@ -749,7 +764,7 @@ class DecisionTreeClassifier(ClassifierMixin, BaseDecisionTree): n_outputs_ : int The number of outputs when ``fit`` is performed. - tree_ : Tree object + tree_ : Tree The underlying Tree object. Please refer to ``help(sklearn.tree._tree.Tree)`` for attributes of Tree object and :ref:`sphx_glr_auto_examples_tree_plot_unveil_tree_structure.py` @@ -767,13 +782,6 @@ class DecisionTreeClassifier(ClassifierMixin, BaseDecisionTree): reduce memory consumption, the complexity and size of the trees should be controlled by setting those parameter values. - The features are always randomly permuted at each split. Therefore, - the best found split may vary, even with the same training data and - ``max_features=n_features``, if the improvement of the criterion is - identical for several splits enumerated during the search of the best - split. To obtain a deterministic behaviour during fitting, - ``random_state`` has to be fixed. - References ---------- @@ -838,7 +846,7 @@ def fit(self, X, y, sample_weight=None, check_input=True, Parameters ---------- - X : {array-like or sparse matrix} of shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The training input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csc_matrix``. @@ -853,11 +861,12 @@ def fit(self, X, y, sample_weight=None, check_input=True, ignored if they would result in any single class carrying a negative weight in either child node. - check_input : bool, (default=True) + check_input : bool, default=True Allow to bypass several input checking. Don't use this parameter unless you know what you do. - X_idx_sorted : array-like of shape (n_samples, n_features), optional + X_idx_sorted : array-like of shape (n_samples, n_features), \ + default=None The indexes of the sorted training input samples. If many tree are grown on the same dataset, this allows the ordering to be cached between trees. If None, the data will be sorted here. @@ -865,7 +874,7 @@ def fit(self, X, y, sample_weight=None, check_input=True, Returns ------- - self : object + self : DecisionTreeClassifier Fitted estimator. """ @@ -882,24 +891,21 @@ def predict_proba(self, X, check_input=True): The predicted class probability is the fraction of samples of the same class in a leaf. - check_input : boolean, (default=True) - Allow to bypass several input checking. - Don't use this parameter unless you know what you do. - Parameters ---------- - X : array-like or sparse matrix of shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. - check_input : bool - Run check_array on X. + check_input : bool, default=True + Allow to bypass several input checking. + Don't use this parameter unless you know what you do. Returns ------- - proba : array of shape (n_samples, n_classes), or a list of n_outputs \ - such arrays if n_outputs > 1. + proba : ndarray of shape (n_samples, n_classes) or list of n_outputs \ + such arrays if n_outputs > 1 The class probabilities of the input samples. The order of the classes corresponds to that in the attribute :term:`classes_`. """ @@ -932,15 +938,15 @@ def predict_log_proba(self, X): Parameters ---------- - X : array-like or sparse matrix of shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. Returns ------- - proba : array of shape (n_samples, n_classes), or a list of n_outputs \ - such arrays if n_outputs > 1. + proba : ndarray of shape (n_samples, n_classes) or list of n_outputs \ + such arrays if n_outputs > 1 The class log-probabilities of the input samples. The order of the classes corresponds to that in the attribute :term:`classes_`. """ @@ -963,7 +969,7 @@ class DecisionTreeRegressor(RegressorMixin, BaseDecisionTree): Parameters ---------- - criterion : str, optional (default="mse") + criterion : {"mse", "friedman_mse", "mae"}, default="mse" The function to measure the quality of a split. Supported criteria are "mse" for the mean squared error, which is equal to variance reduction as feature selection criterion and minimizes the L2 loss @@ -975,17 +981,17 @@ class DecisionTreeRegressor(RegressorMixin, BaseDecisionTree): .. versionadded:: 0.18 Mean Absolute Error (MAE) criterion. - splitter : str, optional (default="best") + splitter : {"best", "random"}, default="best" The strategy used to choose the split at each node. Supported strategies are "best" to choose the best split and "random" to choose the best random split. - max_depth : int or None, optional (default=None) + max_depth : int, default=None The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples. - min_samples_split : int, float, optional (default=2) + min_samples_split : int or float, default=2 The minimum number of samples required to split an internal node: - If int, then consider `min_samples_split` as the minimum number. @@ -996,7 +1002,7 @@ class DecisionTreeRegressor(RegressorMixin, BaseDecisionTree): .. versionchanged:: 0.18 Added float values for fractions. - min_samples_leaf : int, float, optional (default=1) + min_samples_leaf : int or float, default=1 The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least ``min_samples_leaf`` training samples in each of the left and @@ -1011,12 +1017,12 @@ class DecisionTreeRegressor(RegressorMixin, BaseDecisionTree): .. versionchanged:: 0.18 Added float values for fractions. - min_weight_fraction_leaf : float, optional (default=0.) + min_weight_fraction_leaf : float, default=0.0 The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. - max_features : int, float, str or None, optional (default=None) + max_features : int, float or {"auto", "sqrt", "log2"}, default=None The number of features to consider when looking for the best split: - If int, then consider `max_features` features at each split. @@ -1032,18 +1038,24 @@ class DecisionTreeRegressor(RegressorMixin, BaseDecisionTree): valid partition of the node samples is found, even if it requires to effectively inspect more than ``max_features`` features. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - max_leaf_nodes : int or None, optional (default=None) + random_state : int, RandomState instance, default=None + Controls the randomness of the estimator. The features are always + randomly permuted at each split, even if ``splitter`` is set to + ``"best"``. When ``max_features < n_features``, the algorithm will + select ``max_features`` at random at each split before finding the best + split among them. But the best found split may vary across different + runs, even if ``max_features=n_features``. That is the case, if the + improvement of the criterion is identical for several splits and one + split has to be selected at random. To obtain a deterministic behaviour + during fitting, ``random_state`` has to be fixed to an integer. + See :term:`Glossary ` for details. + + max_leaf_nodes : int, default=None Grow a tree with ``max_leaf_nodes`` in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. - min_impurity_decrease : float, optional (default=0.) + min_impurity_decrease : float, default=0.0 A node will be split if this split induces a decrease of the impurity greater than or equal to this value. @@ -1061,14 +1073,14 @@ class DecisionTreeRegressor(RegressorMixin, BaseDecisionTree): .. versionadded:: 0.19 - min_impurity_split : float, (default=1e-7) + min_impurity_split : float, (default=0) Threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf. .. deprecated:: 0.19 ``min_impurity_split`` has been deprecated in favor of ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` will change from 1e-7 to 0 in 0.23 and it + ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it will be removed in 0.25. Use ``min_impurity_decrease`` instead. presort : deprecated, default='deprecated' @@ -1076,7 +1088,7 @@ class DecisionTreeRegressor(RegressorMixin, BaseDecisionTree): .. deprecated:: 0.22 - ccp_alpha : non-negative float, optional (default=0.0) + ccp_alpha : non-negative float, default=0.0 Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ``ccp_alpha`` will be chosen. By default, no pruning is performed. See @@ -1093,7 +1105,11 @@ class DecisionTreeRegressor(RegressorMixin, BaseDecisionTree): (normalized) total reduction of the criterion brought by that feature. It is also known as the Gini importance [4]_. - max_features_ : int, + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + + max_features_ : int The inferred value of max_features. n_features_ : int @@ -1102,7 +1118,7 @@ class DecisionTreeRegressor(RegressorMixin, BaseDecisionTree): n_outputs_ : int The number of outputs when ``fit`` is performed. - tree_ : Tree object + tree_ : Tree The underlying Tree object. Please refer to ``help(sklearn.tree._tree.Tree)`` for attributes of Tree object and :ref:`sphx_glr_auto_examples_tree_plot_unveil_tree_structure.py` @@ -1120,13 +1136,6 @@ class DecisionTreeRegressor(RegressorMixin, BaseDecisionTree): reduce memory consumption, the complexity and size of the trees should be controlled by setting those parameter values. - The features are always randomly permuted at each split. Therefore, - the best found split may vary, even with the same training data and - ``max_features=n_features``, if the improvement of the criterion is - identical for several splits enumerated during the search of the best - split. To obtain a deterministic behaviour during fitting, - ``random_state`` has to be fixed. - References ---------- @@ -1189,7 +1198,7 @@ def fit(self, X, y, sample_weight=None, check_input=True, Parameters ---------- - X : {array-like or sparse matrix} of shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) The training input samples. Internally, it will be converted to ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csc_matrix``. @@ -1203,11 +1212,12 @@ def fit(self, X, y, sample_weight=None, check_input=True, that would create child nodes with net zero or negative weight are ignored while searching for a split in each node. - check_input : bool, (default=True) + check_input : bool, default=True Allow to bypass several input checking. Don't use this parameter unless you know what you do. - X_idx_sorted : array-like of shape (n_samples, n_features), optional + X_idx_sorted : array-like of shape (n_samples, n_features), \ + default=None The indexes of the sorted training input samples. If many tree are grown on the same dataset, this allows the ordering to be cached between trees. If None, the data will be sorted here. @@ -1215,7 +1225,7 @@ def fit(self, X, y, sample_weight=None, check_input=True, Returns ------- - self : object + self : DecisionTreeRegressor Fitted estimator. """ @@ -1242,6 +1252,31 @@ def n_classes_(self): warnings.warn(msg, FutureWarning) return np.array([1] * self.n_outputs_, dtype=np.intp) + def _compute_partial_dependence_recursion(self, grid, target_features): + """Fast partial dependence computation. + + Parameters + ---------- + grid : ndarray of shape (n_samples, n_target_features) + The grid points on which the partial dependence should be + evaluated. + target_features : ndarray of shape (n_target_features) + The set of target features for which the partial dependence + should be evaluated. + + Returns + ------- + averaged_predictions : ndarray of shape (n_samples,) + The value of the partial dependence function on each grid point. + """ + grid = np.asarray(grid, dtype=DTYPE, order='C') + averaged_predictions = np.zeros(shape=grid.shape[0], + dtype=np.float64, order='C') + + self.tree_.compute_partial_dependence( + grid, target_features, averaged_predictions) + return averaged_predictions + class ExtraTreeClassifier(DecisionTreeClassifier): """An extremely randomized tree classifier. @@ -1259,21 +1294,21 @@ class ExtraTreeClassifier(DecisionTreeClassifier): Parameters ---------- - criterion : str, optional (default="gini") + criterion : {"gini", "entropy"}, default="gini" The function to measure the quality of a split. Supported criteria are "gini" for the Gini impurity and "entropy" for the information gain. - splitter : str, optional (default="random") + splitter : {"random", "best"}, default="random" The strategy used to choose the split at each node. Supported strategies are "best" to choose the best split and "random" to choose the best random split. - max_depth : int or None, optional (default=None) + max_depth : int, default=None The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples. - min_samples_split : int, float, optional (default=2) + min_samples_split : int or float, default=2 The minimum number of samples required to split an internal node: - If int, then consider `min_samples_split` as the minimum number. @@ -1284,7 +1319,7 @@ class ExtraTreeClassifier(DecisionTreeClassifier): .. versionchanged:: 0.18 Added float values for fractions. - min_samples_leaf : int, float, optional (default=1) + min_samples_leaf : int or float, default=1 The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least ``min_samples_leaf`` training samples in each of the left and @@ -1299,12 +1334,12 @@ class ExtraTreeClassifier(DecisionTreeClassifier): .. versionchanged:: 0.18 Added float values for fractions. - min_weight_fraction_leaf : float, optional (default=0.) + min_weight_fraction_leaf : float, default=0.0 The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. - max_features : int, float, str or None, optional (default="auto") + max_features : int, float, {"auto", "sqrt", "log2"} or None, default="auto" The number of features to consider when looking for the best split: - If int, then consider `max_features` features at each split. @@ -1320,18 +1355,16 @@ class ExtraTreeClassifier(DecisionTreeClassifier): valid partition of the node samples is found, even if it requires to effectively inspect more than ``max_features`` features. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance, default=None + Used to pick randomly the `max_features` used at each split. + See :term:`Glossary ` for details. - max_leaf_nodes : int or None, optional (default=None) + max_leaf_nodes : int, default=None Grow a tree with ``max_leaf_nodes`` in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. - min_impurity_decrease : float, optional (default=0.) + min_impurity_decrease : float, default=0.0 A node will be split if this split induces a decrease of the impurity greater than or equal to this value. @@ -1349,19 +1382,19 @@ class ExtraTreeClassifier(DecisionTreeClassifier): .. versionadded:: 0.19 - min_impurity_split : float, (default=1e-7) + min_impurity_split : float, (default=0) Threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf. .. deprecated:: 0.19 ``min_impurity_split`` has been deprecated in favor of ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` will change from 1e-7 to 0 in 0.23 and it + ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it will be removed in 0.25. Use ``min_impurity_decrease`` instead. - class_weight : dict, list of dicts, "balanced" or None, default=None + class_weight : dict, list of dict or "balanced", default=None Weights associated with classes in the form ``{class_label: weight}``. - If not given, all classes are supposed to have weight one. For + If None, all classes are supposed to have weight one. For multi-output problems, a list of dicts can be provided in the same order as the columns of y. @@ -1380,7 +1413,7 @@ class ExtraTreeClassifier(DecisionTreeClassifier): Note that these weights will be multiplied with sample_weight (passed through the fit method) if sample_weight is specified. - ccp_alpha : non-negative float, optional (default=0.0) + ccp_alpha : non-negative float, default=0.0 Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ``ccp_alpha`` will be chosen. By default, no pruning is performed. See @@ -1390,21 +1423,28 @@ class ExtraTreeClassifier(DecisionTreeClassifier): Attributes ---------- - classes_ : array of shape (n_classes,) or a list of such arrays + classes_ : ndarray of shape (n_classes,) or list of ndarray The classes labels (single output problem), or a list of arrays of class labels (multi-output problem). - max_features_ : int, + max_features_ : int The inferred value of max_features. - n_classes_ : int or list + n_classes_ : int or list of int The number of classes (for single output problems), or a list containing the number of classes for each output (for multi-output problems). feature_importances_ : ndarray of shape (n_features,) - Return the feature importances (the higher, the more important the - feature). + The impurity-based feature importances. + The higher, the more important the feature. + The importance of a feature is computed as the (normalized) + total reduction of the criterion brought by that feature. It is also + known as the Gini importance. + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. n_features_ : int The number of features when ``fit`` is performed. @@ -1412,7 +1452,7 @@ class ExtraTreeClassifier(DecisionTreeClassifier): n_outputs_ : int The number of outputs when ``fit`` is performed. - tree_ : Tree object + tree_ : Tree The underlying Tree object. Please refer to ``help(sklearn.tree._tree.Tree)`` for attributes of Tree object and :ref:`sphx_glr_auto_examples_tree_plot_unveil_tree_structure.py` @@ -1420,8 +1460,9 @@ class ExtraTreeClassifier(DecisionTreeClassifier): See Also -------- - ExtraTreeRegressor, sklearn.ensemble.ExtraTreesClassifier, - sklearn.ensemble.ExtraTreesRegressor + ExtraTreeRegressor : An extremely randomized tree regressor. + sklearn.ensemble.ExtraTreesClassifier : An extra-trees classifier. + sklearn.ensemble.ExtraTreesRegressor : An extra-trees regressor. Notes ----- @@ -1483,7 +1524,7 @@ class ExtraTreeRegressor(DecisionTreeRegressor): Parameters ---------- - criterion : str, optional (default="mse") + criterion : {"mse", "friedman_mse", "mae"}, default="mse" The function to measure the quality of a split. Supported criteria are "mse" for the mean squared error, which is equal to variance reduction as feature selection criterion, and "mae" for the mean @@ -1492,17 +1533,17 @@ class ExtraTreeRegressor(DecisionTreeRegressor): .. versionadded:: 0.18 Mean Absolute Error (MAE) criterion. - splitter : str, optional (default="random") + splitter : {"random", "best"}, default="random" The strategy used to choose the split at each node. Supported strategies are "best" to choose the best split and "random" to choose the best random split. - max_depth : int or None, optional (default=None) + max_depth : int, default=None The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples. - min_samples_split : int, float, optional (default=2) + min_samples_split : int or float, default=2 The minimum number of samples required to split an internal node: - If int, then consider `min_samples_split` as the minimum number. @@ -1513,7 +1554,7 @@ class ExtraTreeRegressor(DecisionTreeRegressor): .. versionchanged:: 0.18 Added float values for fractions. - min_samples_leaf : int, float, optional (default=1) + min_samples_leaf : int or float, default=1 The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least ``min_samples_leaf`` training samples in each of the left and @@ -1528,12 +1569,12 @@ class ExtraTreeRegressor(DecisionTreeRegressor): .. versionchanged:: 0.18 Added float values for fractions. - min_weight_fraction_leaf : float, optional (default=0.) + min_weight_fraction_leaf : float, default=0.0 The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. - max_features : int, float, str or None, optional (default="auto") + max_features : int, float, {"auto", "sqrt", "log2"} or None, default="auto" The number of features to consider when looking for the best split: - If int, then consider `max_features` features at each split. @@ -1549,13 +1590,11 @@ class ExtraTreeRegressor(DecisionTreeRegressor): valid partition of the node samples is found, even if it requires to effectively inspect more than ``max_features`` features. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance, default=None + Used to pick randomly the `max_features` used at each split. + See :term:`Glossary ` for details. - min_impurity_decrease : float, optional (default=0.) + min_impurity_decrease : float, default=0.0 A node will be split if this split induces a decrease of the impurity greater than or equal to this value. @@ -1573,22 +1612,22 @@ class ExtraTreeRegressor(DecisionTreeRegressor): .. versionadded:: 0.19 - min_impurity_split : float, (default=1e-7) + min_impurity_split : float, (default=0) Threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf. .. deprecated:: 0.19 ``min_impurity_split`` has been deprecated in favor of ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` will change from 1e-7 to 0 in 0.23 and it + ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it will be removed in 0.25. Use ``min_impurity_decrease`` instead. - max_leaf_nodes : int or None, optional (default=None) + max_leaf_nodes : int, default=None Grow a tree with ``max_leaf_nodes`` in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. - ccp_alpha : non-negative float, optional (default=0.0) + ccp_alpha : non-negative float, default=0.0 Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ``ccp_alpha`` will be chosen. By default, no pruning is performed. See @@ -1598,16 +1637,24 @@ class ExtraTreeRegressor(DecisionTreeRegressor): Attributes ---------- - max_features_ : int, + max_features_ : int The inferred value of max_features. n_features_ : int The number of features when ``fit`` is performed. + feature_importances_ : ndarray of shape (n_features,) + Return impurity-based feature importances (the higher, the more + important the feature). + + Warning: impurity-based feature importances can be misleading for + high cardinality features (many unique values). See + :func:`sklearn.inspection.permutation_importance` as an alternative. + n_outputs_ : int The number of outputs when ``fit`` is performed. - tree_ : Tree object + tree_ : Tree The underlying Tree object. Please refer to ``help(sklearn.tree._tree.Tree)`` for attributes of Tree object and :ref:`sphx_glr_auto_examples_tree_plot_unveil_tree_structure.py` @@ -1615,8 +1662,9 @@ class ExtraTreeRegressor(DecisionTreeRegressor): See Also -------- - ExtraTreeClassifier, sklearn.ensemble.ExtraTreesClassifier, - sklearn.ensemble.ExtraTreesRegressor + ExtraTreeClassifier : An extremely randomized tree classifier. + sklearn.ensemble.ExtraTreesClassifier : An extra-trees classifier. + sklearn.ensemble.ExtraTreesRegressor : An extra-trees regressor. Notes ----- @@ -1631,6 +1679,21 @@ class ExtraTreeRegressor(DecisionTreeRegressor): .. [1] P. Geurts, D. Ernst., and L. Wehenkel, "Extremely randomized trees", Machine Learning, 63(1), 3-42, 2006. + + Examples + -------- + >>> from sklearn.datasets import load_boston + >>> from sklearn.model_selection import train_test_split + >>> from sklearn.ensemble import BaggingRegressor + >>> from sklearn.tree import ExtraTreeRegressor + >>> X, y = load_boston(return_X_y=True) + >>> X_train, X_test, y_train, y_test = train_test_split( + ... X, y, random_state=0) + >>> extra_tree = ExtraTreeRegressor(random_state=0) + >>> reg = BaggingRegressor(extra_tree, random_state=0).fit( + ... X_train, y_train) + >>> reg.score(X_test, y_test) + 0.7447... """ def __init__(self, criterion="mse", diff --git a/sklearn/tree/_export.py b/sklearn/tree/_export.py index 826f4345298d2..3197995818f81 100644 --- a/sklearn/tree/_export.py +++ b/sklearn/tree/_export.py @@ -24,6 +24,8 @@ from ._reingold_tilford import buchheim, Tree from . import DecisionTreeClassifier +import warnings + def _color_brew(n): """Generate n colors with equally spaced hues. @@ -78,7 +80,7 @@ def __repr__(self): def plot_tree(decision_tree, max_depth=None, feature_names=None, class_names=None, label='all', filled=False, impurity=True, node_ids=False, - proportion=False, rotate=False, rounded=False, + proportion=False, rotate='deprecated', rounded=False, precision=3, ax=None, fontsize=None): """Plot a decision tree. @@ -131,7 +133,12 @@ def plot_tree(decision_tree, max_depth=None, feature_names=None, to be proportions and percentages respectively. rotate : bool, optional (default=False) - When set to ``True``, orient tree left to right rather than top-down. + This parameter has no effect on the matplotlib tree visualisation and + it is kept here for backward compatibility. + + .. deprecated:: 0.23 + ``rotate`` is deprecated in 0.23 and will be removed in 0.25. + rounded : bool, optional (default=False) When set to ``True``, draw node boxes with rounded corners and use @@ -167,6 +174,14 @@ def plot_tree(decision_tree, max_depth=None, feature_names=None, [Text(251.5,345.217,'X[3] <= 0.8... """ + + check_is_fitted(decision_tree) + + if rotate != 'deprecated': + warnings.warn(("'rotate' has no effect and is deprecated in 0.23. " + "It will be removed in 0.25."), + FutureWarning) + exporter = _MPLTreeExporter( max_depth=max_depth, feature_names=feature_names, class_names=class_names, label=label, filled=filled, @@ -559,6 +574,7 @@ def _make_tree(self, node_id, et, criterion, depth=0): def export(self, decision_tree, ax=None): import matplotlib.pyplot as plt from matplotlib.text import Annotation + if ax is None: ax = plt.gca() ax.clear() diff --git a/sklearn/tree/_tree.pyx b/sklearn/tree/_tree.pyx index bbe2c8a796578..e6b68f0a76534 100644 --- a/sklearn/tree/_tree.pyx +++ b/sklearn/tree/_tree.pyx @@ -792,7 +792,7 @@ cdef class Tree: raise ValueError("X.dtype should be np.float32, got %s" % X.dtype) # Extract input - cdef DTYPE_t[:, :] X_ndarray = X + cdef const DTYPE_t[:, :] X_ndarray = X cdef SIZE_t n_samples = X.shape[0] # Initialize output @@ -912,7 +912,7 @@ cdef class Tree: raise ValueError("X.dtype should be np.float32, got %s" % X.dtype) # Extract input - cdef DTYPE_t[:, :] X_ndarray = X + cdef const DTYPE_t[:, :] X_ndarray = X cdef SIZE_t n_samples = X.shape[0] # Initialize output diff --git a/sklearn/tree/setup.py b/sklearn/tree/setup.py index 2b9819795b74b..079ae9d869075 100644 --- a/sklearn/tree/setup.py +++ b/sklearn/tree/setup.py @@ -31,10 +31,6 @@ def configuration(parent_package="", top_path=None): extra_compile_args=["-O3"]) config.add_subpackage("tests") - config.add_data_files("_criterion.pxd") - config.add_data_files("_splitter.pxd") - config.add_data_files("_tree.pxd") - config.add_data_files("_utils.pxd") return config diff --git a/sklearn/tree/tests/test_export.py b/sklearn/tree/tests/test_export.py index 8122b2096dad0..ad49f81fcf9ac 100644 --- a/sklearn/tree/tests/test_export.py +++ b/sklearn/tree/tests/test_export.py @@ -303,11 +303,11 @@ def test_precision(): # check impurity for finding in finditer(pattern, dot_data): assert (len(search(r"\.\d+", finding.group()).group()) == - precision + 1) + precision + 1) # check threshold for finding in finditer(r"<= \d+\.\d+", dot_data): assert (len(search(r"\.\d+", finding.group()).group()) == - precision + 1) + precision + 1) def test_export_text_errors(): @@ -448,3 +448,23 @@ def test_plot_tree_gini(pyplot): "samples = 6\nvalue = [3, 3]") assert nodes[1].get_text() == "gini = 0.0\nsamples = 3\nvalue = [3, 0]" assert nodes[2].get_text() == "gini = 0.0\nsamples = 3\nvalue = [0, 3]" + + +# FIXME: to be removed in 0.25 +def test_plot_tree_rotate_deprecation(pyplot): + tree = DecisionTreeClassifier() + tree.fit(X, y) + # test that a warning is raised when rotate is used. + match = ("'rotate' has no effect and is deprecated in 0.23. " + "It will be removed in 0.25.") + with pytest.warns(FutureWarning, match=match): + plot_tree(tree, rotate=True) + + +def test_not_fitted_tree(pyplot): + + # Testing if not fitted tree throws the correct error + clf = DecisionTreeRegressor() + out = StringIO() + with pytest.raises(NotFittedError): + plot_tree(clf, out) diff --git a/sklearn/tree/tests/test_reingold_tilford.py b/sklearn/tree/tests/test_reingold_tilford.py index dfab29d0705c0..6494536004333 100644 --- a/sklearn/tree/tests/test_reingold_tilford.py +++ b/sklearn/tree/tests/test_reingold_tilford.py @@ -43,8 +43,8 @@ def walk_tree(draw_tree): # we could also do it quicker using defaultdicts.. depth = 0 while True: - x_at_this_depth = [coordinates[0] for node in coordinates - if coordinates[1] == depth] + x_at_this_depth = [node[0] for node in coordinates + if node[1] == depth] if not x_at_this_depth: # reached all leafs break diff --git a/sklearn/tree/tests/test_tree.py b/sklearn/tree/tests/test_tree.py index dcd9d4c01a8ec..1149ceb8678d9 100644 --- a/sklearn/tree/tests/test_tree.py +++ b/sklearn/tree/tests/test_tree.py @@ -23,6 +23,7 @@ from sklearn.utils._testing import assert_almost_equal from sklearn.utils._testing import assert_warns from sklearn.utils._testing import assert_warns_message +from sklearn.utils._testing import create_memmap_backed_data from sklearn.utils._testing import ignore_warnings from sklearn.utils._testing import TempMemmap @@ -803,7 +804,7 @@ def test_min_impurity_split(): est = TreeEstimator(max_leaf_nodes=max_leaf_nodes, random_state=0) assert est.min_impurity_split is None, ( - "Failed, min_impurity_split = {0} > 1e-7".format( + "Failed, min_impurity_split = {0} != None".format( est.min_impurity_split)) try: assert_warns(FutureWarning, est.fit, X, y) @@ -1124,14 +1125,6 @@ def test_sample_weight_invalid(): with pytest.raises(TypeError, match=expected_err): clf.fit(X, y, sample_weight=sample_weight) - sample_weight = np.ones(101) - with pytest.raises(ValueError): - clf.fit(X, y, sample_weight=sample_weight) - - sample_weight = np.ones(99) - with pytest.raises(ValueError): - clf.fit(X, y, sample_weight=sample_weight) - def check_class_weights(name): """Check class_weights resemble sample_weights behavior.""" @@ -1823,15 +1816,6 @@ def test_empty_leaf_infinite_threshold(): assert len(empty_leaf) == 0 -def test_decision_tree_memmap(): - # check that decision trees supports read-only buffer (#13626) - X = np.random.RandomState(0).random_sample((10, 2)).astype(np.float32) - y = np.zeros(10) - - with TempMemmap((X, y)) as (X_read_only, y_read_only): - DecisionTreeClassifier().fit(X_read_only, y_read_only) - - @pytest.mark.parametrize("criterion", CLF_CRITERIONS) @pytest.mark.parametrize( "dataset", sorted(set(DATASETS.keys()) - {"reg_small", "boston"})) @@ -1963,3 +1947,21 @@ def test_classes_deprecated(): with pytest.warns(FutureWarning, match=match): assert len(clf.n_classes_) == clf.n_outputs_ + + +def check_apply_path_readonly(name): + X_readonly = create_memmap_backed_data(X_small.astype(tree._tree.DTYPE, + copy=False)) + y_readonly = create_memmap_backed_data(np.array(y_small, + dtype=tree._tree.DTYPE)) + est = ALL_TREES[name]() + est.fit(X_readonly, y_readonly) + assert_array_equal(est.predict(X_readonly), + est.predict(X_small)) + assert_array_equal(est.decision_path(X_readonly).todense(), + est.decision_path(X_small).todense()) + + +@pytest.mark.parametrize("name", ALL_TREES) +def test_apply_path_readonly_all_trees(name): + check_apply_path_readonly(name) diff --git a/sklearn/utils/__init__.py b/sklearn/utils/__init__.py index 4d4ef606341ca..4b69365339389 100644 --- a/sklearn/utils/__init__.py +++ b/sklearn/utils/__init__.py @@ -3,6 +3,7 @@ """ import pkgutil import inspect +from importlib import import_module from operator import itemgetter from collections.abc import Sequence from contextlib import contextmanager @@ -12,6 +13,7 @@ import platform import struct import timeit +from pathlib import Path import warnings import numpy as np @@ -39,29 +41,6 @@ parallel_backend = _joblib.parallel_backend register_parallel_backend = _joblib.register_parallel_backend -# deprecate the joblib API in sklearn in favor of using directly joblib -msg = ("deprecated in version 0.20.1 to be removed in version 0.23. " - "Please import this functionality directly from joblib, which can " - "be installed with: pip install joblib.") -deprecate = deprecated(msg) - -delayed = deprecate(_joblib.delayed) -cpu_count = deprecate(_joblib.cpu_count) -hash = deprecate(_joblib.hash) -effective_n_jobs = deprecate(_joblib.effective_n_jobs) - - -# for classes, deprecated will change the object in _joblib module so we need -# to subclass them. -@deprecate -class Memory(_joblib.Memory): - pass - - -@deprecate -class Parallel(_joblib.Parallel): - pass - __all__ = ["murmurhash3_32", "as_float_array", "assert_all_finite", "check_array", @@ -70,8 +49,7 @@ class Parallel(_joblib.Parallel): "column_or_1d", "safe_indexing", "check_consistent_length", "check_X_y", "check_scalar", 'indexable', "check_symmetric", "indices_to_mask", "deprecated", - "cpu_count", "Parallel", "Memory", "delayed", "parallel_backend", - "register_parallel_backend", "hash", "effective_n_jobs", + "parallel_backend", "register_parallel_backend", "resample", "shuffle", "check_matplotlib_support", "all_estimators", ] @@ -80,10 +58,14 @@ class Parallel(_joblib.Parallel): class Bunch(dict): - """Container object for datasets + """Container object exposing keys as attributes - Dictionary-like object that exposes its keys as attributes. + Bunch objects are sometimes used as an output for functions and methods. + They extend dictionaries by enabling values to be accessed by key, + `bunch["value_key"]`, or by an attribute, `bunch.value_key`. + Examples + -------- >>> b = Bunch(a=1, b=2) >>> b['b'] 2 @@ -95,7 +77,6 @@ class Bunch(dict): >>> b.c = 6 >>> b['c'] 6 - """ def __init__(self, **kwargs): @@ -438,7 +419,7 @@ def _get_column_indices(X, key): return np.atleast_1d(idx).tolist() elif key_dtype == 'str': try: - all_columns = list(X.columns) + all_columns = X.columns except AttributeError: raise ValueError("Specifying the columns using strings is only " "supported for pandas DataFrames") @@ -447,10 +428,10 @@ def _get_column_indices(X, key): elif isinstance(key, slice): start, stop = key.start, key.stop if start is not None: - start = all_columns.index(start) + start = all_columns.get_loc(start) if stop is not None: # pandas indexing with strings is endpoint included - stop = all_columns.index(stop) + 1 + stop = all_columns.get_loc(stop) + 1 else: stop = n_columns + 1 return list(range(n_columns)[slice(start, stop)]) @@ -458,13 +439,18 @@ def _get_column_indices(X, key): columns = list(key) try: - column_indices = [all_columns.index(col) for col in columns] - except ValueError as e: - if 'not in list' in str(e): - raise ValueError( - "A given column is not a column of the dataframe" - ) from e - raise + column_indices = [] + for col in columns: + col_idx = all_columns.get_loc(col) + if not isinstance(col_idx, numbers.Integral): + raise ValueError(f"Selected columns, {columns}, are not " + "unique in dataframe") + column_indices.append(col_idx) + + except KeyError as e: + raise ValueError( + "A given column is not a column of the dataframe" + ) from e return column_indices else: @@ -498,11 +484,10 @@ def resample(*arrays, **options): arrays. random_state : int, RandomState instance or None, optional (default=None) - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. + Determines random number generation for shuffling + the data. + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary `. stratify : array-like or None (default=None) If not None, data is split in a stratified fashion, using this as @@ -643,11 +628,10 @@ def shuffle(*arrays, **options): Other Parameters ---------------- random_state : int, RandomState instance or None, optional (default=None) - The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. + Determines random number generation for shuffling + the data. + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary `. n_samples : int, None by default Number of samples to generate. If left to None this is @@ -770,6 +754,12 @@ def gen_batches(n, batch_size, min_batch_size=0): >>> list(gen_batches(7, 3, min_batch_size=2)) [slice(0, 3, None), slice(3, 7, None)] """ + if not isinstance(batch_size, numbers.Integral): + raise TypeError("gen_batches got batch_size=%s, must be an" + " integer" % batch_size) + if batch_size <= 0: + raise ValueError("gen_batches got batch_size=%s, must be" + " positive" % batch_size) start = 0 for _ in range(int(n // batch_size)): end = start + batch_size @@ -841,6 +831,45 @@ def tosequence(x): return list(x) +def _to_object_array(sequence): + """Convert sequence to a 1-D NumPy array of object dtype. + + numpy.array constructor has a similar use but it's output + is ambiguous. It can be 1-D NumPy array of object dtype if + the input is a ragged array, but if the input is a list of + equal length arrays, then the output is a 2D numpy.array. + _to_object_array solves this ambiguity by guarantying that + the output is a 1-D NumPy array of objects for any input. + + Parameters + ---------- + sequence : array-like of shape (n_elements,) + The sequence to be converted. + + Returns + ------- + out : ndarray of shape (n_elements,), dtype=object + The converted sequence into a 1-D NumPy array of object dtype. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.utils import _to_object_array + >>> _to_object_array([np.array([0]), np.array([1])]) + array([array([0]), array([1])], dtype=object) + >>> _to_object_array([np.array([0]), np.array([1, 2])]) + array([array([0]), array([1, 2])], dtype=object) + >>> np.array([np.array([0]), np.array([1])]) + array([[0], + [1]]) + >>> np.array([np.array([0]), np.array([1, 2])]) + array([array([0]), array([1, 2])], dtype=object) + """ + out = np.empty(len(sequence), dtype=object) + out[:] = sequence + return out + + def indices_to_mask(indices, mask_length): """Convert list of indices to boolean mask. @@ -1108,9 +1137,7 @@ def check_pandas_support(caller_name): ) from e -def all_estimators(include_meta_estimators=None, - include_other=None, type_filter=None, - include_dont_test=None): +def all_estimators(type_filter=None): """Get a list of all estimators from sklearn. This function crawls the module and gets all classes that inherit @@ -1120,20 +1147,6 @@ def all_estimators(include_meta_estimators=None, Parameters ---------- - include_meta_estimators : boolean, default=False - Deprecated, ignored. - - .. deprecated:: 0.21 - ``include_meta_estimators`` has been deprecated and has no effect in - 0.21 and will be removed in 0.23. - - include_other : boolean, default=False - Deprecated, ignored. - - .. deprecated:: 0.21 - ``include_other`` has been deprecated and has not effect in 0.21 and - will be removed in 0.23. - type_filter : string, list of string, or None, default=None Which kind of estimators should be returned. If None, no filter is applied and all estimators are returned. Possible values are @@ -1141,13 +1154,6 @@ def all_estimators(include_meta_estimators=None, estimators only of these specific types, or a list of these to get the estimators that fit at least one of the types. - include_dont_test : boolean, default=False - Deprecated, ignored. - - .. deprecated:: 0.21 - ``include_dont_test`` has been deprecated and has no effect in 0.21 - and will be removed in 0.23. - Returns ------- estimators : list of tuples @@ -1155,7 +1161,6 @@ def all_estimators(include_meta_estimators=None, and ``class`` is the actuall type of the class. """ # lazy import to avoid circular imports from sklearn.base - import sklearn from ._testing import ignore_warnings from ..base import (BaseEstimator, ClassifierMixin, RegressorMixin, TransformerMixin, ClusterMixin) @@ -1167,36 +1172,30 @@ def is_abstract(c): return False return True - if include_other is not None: - warnings.warn("include_other was deprecated in version 0.21," - " has no effect and will be removed in 0.23", - DeprecationWarning) - - if include_dont_test is not None: - warnings.warn("include_dont_test was deprecated in version 0.21," - " has no effect and will be removed in 0.23", - DeprecationWarning) - - if include_meta_estimators is not None: - warnings.warn("include_meta_estimators was deprecated in version 0.21," - " has no effect and will be removed in 0.23", - DeprecationWarning) - all_classes = [] - # get parent folder - path = sklearn.__path__ - for importer, modname, ispkg in pkgutil.walk_packages( - path=path, prefix='sklearn.', onerror=lambda x: None): - if ".tests." in modname or "externals" in modname: - continue - if IS_PYPY and ('_svmlight_format' in modname or - 'feature_extraction._hashing' in modname): - continue - # Ignore deprecation warnings triggered at import time. - with ignore_warnings(category=FutureWarning): - module = __import__(modname, fromlist="dummy") - classes = inspect.getmembers(module, inspect.isclass) - all_classes.extend(classes) + modules_to_ignore = {"tests", "externals", "setup", "conftest"} + root = str(Path(__file__).parent.parent) # sklearn package + # Ignore deprecation warnings triggered at import time and from walking + # packages + with ignore_warnings(category=FutureWarning): + for importer, modname, ispkg in pkgutil.walk_packages( + path=[root], prefix='sklearn.'): + mod_parts = modname.split(".") + if (any(part in modules_to_ignore for part in mod_parts) + or '._' in modname): + continue + module = import_module(modname) + classes = inspect.getmembers(module, inspect.isclass) + classes = [(name, est_cls) for name, est_cls in classes + if not name.startswith("_")] + + # TODO: Remove when FeatureHasher is implemented in PYPY + # Skips FeatureHasher for PYPY + if IS_PYPY and 'feature_extraction' in modname: + classes = [(name, est_cls) for name, est_cls in classes + if name == "FeatureHasher"] + + all_classes.extend(classes) all_classes = set(all_classes) diff --git a/sklearn/utils/_mask.py b/sklearn/utils/_mask.py index 1f660205cdd47..48b2a11c6f87b 100644 --- a/sklearn/utils/_mask.py +++ b/sklearn/utils/_mask.py @@ -5,7 +5,7 @@ def _get_mask(X, value_to_mask): - """Compute the boolean mask X == missing_values.""" + """Compute the boolean mask X == value_to_mask.""" if is_scalar_nan(value_to_mask): if X.dtype.kind == "f": return np.isnan(X) @@ -16,6 +16,4 @@ def _get_mask(X, value_to_mask): # np.isnan does not work on object dtypes. return _object_dtype_isnan(X) else: - # X == value_to_mask with object dtypes does not always perform - # element-wise for old versions of numpy - return np.equal(X, value_to_mask) + return X == value_to_mask diff --git a/sklearn/utils/_mocking.py b/sklearn/utils/_mocking.py index 3edcf8da53a95..cff4183ea9bc4 100644 --- a/sklearn/utils/_mocking.py +++ b/sklearn/utils/_mocking.py @@ -61,6 +61,10 @@ class CheckingClassifier(ClassifierMixin, BaseEstimator): check_X foo_param expected_fit_params + + Attributes + ---------- + classes_ """ def __init__(self, check_y=None, check_X=None, foo_param=0, expected_fit_params=None): @@ -91,6 +95,7 @@ def fit(self, X, y, **fit_params): assert self.check_X(X) if self.check_y is not None: assert self.check_y(y) + self.n_features_in_ = len(X) self.classes_ = np.unique(check_array(y, ensure_2d=False, allow_nd=True)) if self.expected_fit_params: @@ -158,4 +163,4 @@ def predict_proba(self, X): return self.est.predict_proba(X) def _more_tags(self): - return {'_skip_test': True} # pragma: no cover + return {'_skip_test': True} diff --git a/sklearn/utils/_testing.py b/sklearn/utils/_testing.py index 4aaf7c1fd388c..eb6e381f02840 100644 --- a/sklearn/utils/_testing.py +++ b/sklearn/utils/_testing.py @@ -53,8 +53,8 @@ __all__ = ["assert_equal", "assert_not_equal", "assert_raises", - "assert_raises_regexp", "assert_true", - "assert_false", "assert_almost_equal", "assert_array_equal", + "assert_raises_regexp", + "assert_almost_equal", "assert_array_equal", "assert_array_almost_equal", "assert_array_less", "assert_less", "assert_less_equal", "assert_greater", "assert_greater_equal", @@ -85,16 +85,6 @@ # the old name for now assert_raises_regexp = assert_raises_regex -deprecation_message = "'assert_true' is deprecated in version 0.21 " \ - "and will be removed in version 0.23. " \ - "Please use 'assert' instead." -assert_true = deprecated(deprecation_message)(_dummy.assertTrue) - -deprecation_message = "'assert_false' is deprecated in version 0.21 " \ - "and will be removed in version 0.23. " \ - "Please use 'assert' instead." -assert_false = deprecated(deprecation_message)(_dummy.assertFalse) - def assert_warns(warning_class, func, *args, **kw): """Test that a certain warning occurs. @@ -276,8 +266,8 @@ def ignore_warnings(obj=None, category=Warning): ... warnings.warn('buhuhuhu') >>> def nasty_warn(): - ... warnings.warn('buhuhuhu') - ... print(42) + ... warnings.warn('buhuhuhu') + ... print(42) >>> ignore_warnings(nasty_warn)() 42 @@ -438,9 +428,7 @@ def assert_allclose_dense_sparse(x, y, rtol=1e-07, atol=1e-9, err_msg=''): # TODO: Remove in 0.24. This class is now in utils.__init__. -def all_estimators(include_meta_estimators=None, - include_other=None, type_filter=None, - include_dont_test=None): +def all_estimators(type_filter=None): """Get a list of all estimators from sklearn. This function crawls the module and gets all classes that inherit @@ -450,19 +438,6 @@ def all_estimators(include_meta_estimators=None, Parameters ---------- - include_meta_estimators : boolean, default=False - Deprecated, ignored. - - .. deprecated:: 0.21 - ``include_meta_estimators`` has been deprecated and has no effect in - 0.21 and will be removed in 0.23. - - include_other : boolean, default=False - Deprecated, ignored. - - .. deprecated:: 0.21 - ``include_other`` has been deprecated and has not effect in 0.21 and - will be removed in 0.23. type_filter : string, list of string, or None, default=None Which kind of estimators should be returned. If None, no filter is @@ -471,18 +446,11 @@ def all_estimators(include_meta_estimators=None, estimators only of these specific types, or a list of these to get the estimators that fit at least one of the types. - include_dont_test : boolean, default=False - Deprecated, ignored. - - .. deprecated:: 0.21 - ``include_dont_test`` has been deprecated and has no effect in 0.21 - and will be removed in 0.23. - Returns ------- estimators : list of tuples List of (name, class), where ``name`` is the class name as string - and ``class`` is the actuall type of the class. + and ``class`` is the actual type of the class. """ def is_abstract(c): if not(hasattr(c, '__abstractmethods__')): @@ -491,21 +459,6 @@ def is_abstract(c): return False return True - if include_other is not None: - warnings.warn("include_other was deprecated in version 0.21," - " has no effect and will be removed in 0.23", - FutureWarning) - - if include_dont_test is not None: - warnings.warn("include_dont_test was deprecated in version 0.21," - " has no effect and will be removed in 0.23", - FutureWarning) - - if include_meta_estimators is not None: - warnings.warn("include_meta_estimators was deprecated in version 0.21," - " has no effect and will be removed in 0.23", - FutureWarning) - all_classes = [] # get parent folder path = sklearn.__path__ @@ -513,7 +466,7 @@ def is_abstract(c): path=path, prefix='sklearn.', onerror=lambda x: None): if ".tests." in modname or "externals" in modname: continue - if IS_PYPY and ('_svmlight_format' in modname or + if IS_PYPY and ('_svmlight_format_io' in modname or 'feature_extraction._hashing_fast' in modname): continue # Ignore deprecation warnings triggered at import time. @@ -566,10 +519,9 @@ def set_random_state(estimator, random_state=0): estimator : object The estimator random_state : int, RandomState instance or None, optional, default=0 - Pseudo random number generator state. If int, random_state is the seed - used by the random number generator; If RandomState instance, - random_state is the random number generator; If None, the random number - generator is the RandomState instance used by `np.random`. + Pseudo random number generator state. + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary `. """ if "random_state" in estimator.get_params(): estimator.set_params(random_state=random_state) @@ -613,21 +565,6 @@ def set_random_state(estimator, random_state=0): pass -def clean_warning_registry(): - """Clean Python warning registry for easier testing of warning messages. - - When changing warning filters this function is not necessary with - Python3.5+, as __warningregistry__ will be re-set internally. - See https://bugs.python.org/issue4180 and - https://bugs.python.org/issue21724 for more details. - - """ - for mod in sys.modules.values(): - registry = getattr(mod, "__warningregistry__", None) - if registry is not None: - registry.clear() - - def check_skip_network(): if int(os.environ.get('SKLEARN_SKIP_NETWORK_TESTS', 0)): raise SkipTest("Text tutorial requires large dataset download") @@ -912,3 +849,25 @@ def assert_run_python_script(source_code, timeout=60): % e.output.decode('utf-8')) finally: os.unlink(source_file) + + +def _convert_container(container, constructor_name, columns_name=None): + if constructor_name == 'list': + return list(container) + elif constructor_name == 'tuple': + return tuple(container) + elif constructor_name == 'array': + return np.asarray(container) + elif constructor_name == 'sparse': + return sp.sparse.csr_matrix(container) + elif constructor_name == 'dataframe': + pd = pytest.importorskip('pandas') + return pd.DataFrame(container, columns=columns_name) + elif constructor_name == 'series': + pd = pytest.importorskip('pandas') + return pd.Series(container) + elif constructor_name == 'index': + pd = pytest.importorskip('pandas') + return pd.Index(container) + elif constructor_name == 'slice': + return slice(container[0], container[1]) diff --git a/sklearn/utils/_weight_vector.pxd b/sklearn/utils/_weight_vector.pxd index 1f38bb7e0981f..fc1b47a50ef1f 100644 --- a/sklearn/utils/_weight_vector.pxd +++ b/sklearn/utils/_weight_vector.pxd @@ -1,12 +1,5 @@ """Efficient (dense) parameter vector implementation for linear models. """ -cimport numpy as np - - -cdef extern from "math.h": - cdef extern double sqrt(double x) - - cdef class WeightVector(object): cdef double *w_data_ptr cdef double *aw_data_ptr diff --git a/sklearn/utils/deprecation.py b/sklearn/utils/deprecation.py index 2a33f34dfd2b8..7780cac7b52fb 100644 --- a/sklearn/utils/deprecation.py +++ b/sklearn/utils/deprecation.py @@ -109,12 +109,12 @@ def _update_doc(self, olddoc): if self.extra: newdoc = "%s: %s" % (newdoc, self.extra) if olddoc: - newdoc = "%s\n\n%s" % (newdoc, olddoc) + newdoc = "%s\n\n %s" % (newdoc, olddoc) return newdoc def _is_deprecated(func): - """Helper to check if func is wraped by our deprecated decorator""" + """Helper to check if func is wrapped by our deprecated decorator""" closures = getattr(func, '__closure__', []) if closures is None: closures = [] diff --git a/sklearn/utils/estimator_checks.py b/sklearn/utils/estimator_checks.py index 30c668237b371..2cfb06c7994db 100644 --- a/sklearn/utils/estimator_checks.py +++ b/sklearn/utils/estimator_checks.py @@ -80,6 +80,7 @@ def _yield_checks(name, estimator): yield check_sample_weights_pandas_series yield check_sample_weights_not_an_array yield check_sample_weights_list + yield check_sample_weights_shape yield check_sample_weights_invariance yield check_estimators_fit_returns_self yield partial(check_estimators_fit_returns_self, readonly_memmap=True) @@ -117,7 +118,7 @@ def _yield_checks(name, estimator): def _yield_classifier_checks(name, classifier): tags = _safe_tags(classifier) - # test classifiers can handle non-array data + # test classifiers can handle non-array data and pandas objects yield check_classifier_data_not_an_array # test classifiers trained on a single label always return this label yield check_classifiers_one_label @@ -128,6 +129,8 @@ def _yield_classifier_checks(name, classifier): # basic consistency testing yield check_classifiers_train yield partial(check_classifiers_train, readonly_memmap=True) + yield partial(check_classifiers_train, readonly_memmap=True, + X_dtype='float32') yield check_classifiers_regression_target if tags["multilabel"]: yield check_classifiers_multilabel_representation_invariance @@ -174,6 +177,8 @@ def _yield_regressor_checks(name, regressor): # basic testing yield check_regressors_train yield partial(check_regressors_train, readonly_memmap=True) + yield partial(check_regressors_train, readonly_memmap=True, + X_dtype='float32') yield check_regressor_data_not_an_array yield check_estimators_partial_fit_n_features if tags["multioutput"]: @@ -276,6 +281,8 @@ def _yield_all_checks(name, estimator): yield check_dict_unchanged yield check_dont_overwrite_parameters yield check_fit_idempotent + if not tags["no_validation"]: + yield check_n_features_in if tags["requires_positive_X"]: yield check_fit_non_negative @@ -311,8 +318,8 @@ def _set_check_estimator_ids(obj): if not obj.keywords: return obj.func.__name__ - kwstring = "".join(["{}={}".format(k, v) - for k, v in obj.keywords.items()]) + kwstring = ",".join(["{}={}".format(k, v) + for k, v in obj.keywords.items()]) return "{}({})".format(obj.func.__name__, kwstring) if hasattr(obj, "get_params"): with config_context(print_changed_only=True): @@ -351,6 +358,23 @@ def _generate_class_checks(Estimator): yield from _generate_instance_checks(name, estimator) +def _mark_xfail_checks(estimator, check, pytest): + """Mark estimator check pairs with xfail""" + + xfail_checks = _safe_tags(estimator, '_xfail_test') + if not xfail_checks: + return estimator, check + + check_name = _set_check_estimator_ids(check) + msg = xfail_checks.get(check_name, None) + + if msg is None: + return estimator, check + + return pytest.param( + estimator, check, marks=pytest.mark.xfail(reason=msg)) + + def parametrize_with_checks(estimators): """Pytest specific decorator for parametrizing estimator checks. @@ -369,11 +393,17 @@ def parametrize_with_checks(estimators): decorator : `pytest.mark.parametrize` """ import pytest - return pytest.mark.parametrize( - "estimator, check", - chain.from_iterable(check_estimator(estimator, generate_only=True) - for estimator in estimators), - ids=_set_check_estimator_ids) + + checks_generator = chain.from_iterable( + check_estimator(estimator, generate_only=True) + for estimator in estimators) + + checks_with_marks = ( + _mark_xfail_checks(estimator, check, pytest) + for estimator, check in checks_generator) + + return pytest.mark.parametrize("estimator, check", checks_with_marks, + ids=_set_check_estimator_ids) def check_estimator(Estimator, generate_only=False): @@ -479,9 +509,6 @@ def _set_checking_parameters(estimator): # K-Means estimator.set_params(n_init=2) - if hasattr(estimator, "n_components"): - estimator.n_components = 2 - if name == 'TruncatedSVD': # TruncatedSVD doesn't run with n_components = n_features # This is ugly :-/ @@ -763,6 +790,31 @@ def check_sample_weights_list(name, estimator_orig): estimator.fit(X, y, sample_weight=sample_weight) +@ignore_warnings(category=FutureWarning) +def check_sample_weights_shape(name, estimator_orig): + # check that estimators raise an error if sample_weight + # shape mismatches the input + if (has_fit_parameter(estimator_orig, "sample_weight") and + not (hasattr(estimator_orig, "_pairwise") + and estimator_orig._pairwise)): + estimator = clone(estimator_orig) + X = np.array([[1, 3], [1, 3], [1, 3], [1, 3], + [2, 1], [2, 1], [2, 1], [2, 1], + [3, 3], [3, 3], [3, 3], [3, 3], + [4, 1], [4, 1], [4, 1], [4, 1]]) + y = np.array([1, 1, 1, 1, 2, 2, 2, 2, + 1, 1, 1, 1, 2, 2, 2, 2]) + y = _enforce_estimator_tags_y(estimator, y) + + estimator.fit(X, y, sample_weight=np.ones(len(y))) + + assert_raises(ValueError, estimator.fit, X, y, + sample_weight=np.ones(2*len(y))) + + assert_raises(ValueError, estimator.fit, X, y, + sample_weight=np.ones((len(y), 2))) + + @ignore_warnings(category=FutureWarning) def check_sample_weights_invariance(name, estimator_orig): # check that the estimators yield same results for @@ -1031,13 +1083,6 @@ def check_methods_subset_invariance(name, estimator_orig): msg = ("{method} of {name} is not invariant when applied " "to a subset.").format(method=method, name=name) - # TODO remove cases when corrected - if (name, method) in [('NuSVC', 'decision_function'), - ('SparsePCA', 'transform'), - ('MiniBatchSparsePCA', 'transform'), - ('DummyClassifier', 'predict'), - ('BernoulliRBM', 'score_samples')]: - raise SkipTest(msg) if hasattr(estimator, method): result_full, result_by_batch = _apply_on_subsets( @@ -1714,8 +1759,10 @@ def check_classifiers_one_label(name, classifier_orig): @ignore_warnings # Warnings are raised by decision function -def check_classifiers_train(name, classifier_orig, readonly_memmap=False): +def check_classifiers_train(name, classifier_orig, readonly_memmap=False, + X_dtype='float64'): X_m, y_m = make_blobs(n_samples=300, random_state=0) + X_m = X_m.astype(X_dtype) X_m, y_m = shuffle(X_m, y_m, random_state=7) X_m = StandardScaler().fit_transform(X_m) # generate binary problem from multi-class one @@ -2166,8 +2213,10 @@ def check_regressors_int(name, regressor_orig): @ignore_warnings(category=FutureWarning) -def check_regressors_train(name, regressor_orig, readonly_memmap=False): +def check_regressors_train(name, regressor_orig, readonly_memmap=False, + X_dtype=np.float64): X, y = _boston_subset() + X = X.astype(X_dtype) X = _pairwise_estimator_convert_X(X, regressor_orig) y = StandardScaler().fit_transform(y.reshape(-1, 1)) # X is already scaled y = y.ravel() @@ -2238,13 +2287,6 @@ def check_regressors_no_decision_function(name, regressor_orig): @ignore_warnings(category=FutureWarning) def check_class_weight_classifiers(name, classifier_orig): - if name == "NuSVC": - # the sparse version has a parameter that doesn't do anything - raise SkipTest("Not testing NuSVC class weight as it is ignored.") - if name.endswith("NB"): - # NaiveBayes classifiers have a somewhat different interface. - # FIXME SOON! - raise SkipTest if _safe_tags(classifier_orig, 'binary_only'): problems = [2] @@ -2450,7 +2492,9 @@ def check_classifier_data_not_an_array(name, estimator_orig): X = _pairwise_estimator_convert_X(X, estimator_orig) y = [1, 1, 1, 2, 2, 2, 1, 1, 1, 2, 2, 2] y = _enforce_estimator_tags_y(estimator_orig, y) - check_estimators_data_not_an_array(name, estimator_orig, X, y) + for obj_type in ["NotAnArray", "PandasDataframe"]: + check_estimators_data_not_an_array(name, estimator_orig, X, y, + obj_type) @ignore_warnings(category=FutureWarning) @@ -2458,11 +2502,13 @@ def check_regressor_data_not_an_array(name, estimator_orig): X, y = _boston_subset(n_samples=50) X = _pairwise_estimator_convert_X(X, estimator_orig) y = _enforce_estimator_tags_y(estimator_orig, y) - check_estimators_data_not_an_array(name, estimator_orig, X, y) + for obj_type in ["NotAnArray", "PandasDataframe"]: + check_estimators_data_not_an_array(name, estimator_orig, X, y, + obj_type) @ignore_warnings(category=FutureWarning) -def check_estimators_data_not_an_array(name, estimator_orig, X, y): +def check_estimators_data_not_an_array(name, estimator_orig, X, y, obj_type): if name in CROSS_DECOMPOSITION: raise SkipTest("Skipping check_estimators_data_not_an_array " "for cross decomposition module as estimators " @@ -2473,8 +2519,28 @@ def check_estimators_data_not_an_array(name, estimator_orig, X, y): set_random_state(estimator_1) set_random_state(estimator_2) - y_ = _NotAnArray(np.asarray(y)) - X_ = _NotAnArray(np.asarray(X)) + if obj_type not in ["NotAnArray", 'PandasDataframe']: + raise ValueError("Data type {0} not supported".format(obj_type)) + + if obj_type == "NotAnArray": + y_ = _NotAnArray(np.asarray(y)) + X_ = _NotAnArray(np.asarray(X)) + else: + # Here pandas objects (Series and DataFrame) are tested explicitly + # because some estimators may handle them (especially their indexing) + # specially. + try: + import pandas as pd + y_ = np.asarray(y) + if y_.ndim == 1: + y_ = pd.Series(y_) + else: + y_ = pd.DataFrame(y_) + X_ = pd.DataFrame(np.asarray(X)) + + except ImportError: + raise SkipTest("pandas is not installed: not checking estimators " + "for pandas objects.") # fit estimator_1.fit(X_, y_) @@ -2572,6 +2638,22 @@ def _enforce_estimator_tags_y(estimator, y): return y +def _enforce_estimator_tags_x(estimator, X): + # Estimators with a `_pairwise` tag only accept + # X of shape (`n_samples`, `n_samples`) + if hasattr(estimator, '_pairwise'): + X = X.dot(X.T) + # Estimators with `1darray` in `X_types` tag only accept + # X of shape (`n_samples`,) + if '1darray' in _safe_tags(estimator, 'X_types'): + X = X[:, 0] + # Estimators with a `requires_positive_X` tag only accept + # strictly positive data + if _safe_tags(estimator, 'requires_positive_X'): + X -= X.min() + return X + + @ignore_warnings(category=FutureWarning) def check_non_transformer_estimators_n_iter(name, estimator_orig): # Test that estimators that are not transformers with a parameter @@ -2842,3 +2924,41 @@ def check_fit_idempotent(name, estimator_orig): atol=max(tol, 1e-9), rtol=max(tol, 1e-7), err_msg="Idempotency check failed for method {}".format(method) ) + + +def check_n_features_in(name, estimator_orig): + # Make sure that n_features_in_ attribute doesn't exist until fit is + # called, and that its value is correct. + + rng = np.random.RandomState(0) + + estimator = clone(estimator_orig) + set_random_state(estimator) + if 'warm_start' in estimator.get_params(): + estimator.set_params(warm_start=False) + + n_samples = 100 + X = rng.normal(loc=100, size=(n_samples, 2)) + X = _pairwise_estimator_convert_X(X, estimator) + if is_regressor(estimator_orig): + y = rng.normal(size=n_samples) + else: + y = rng.randint(low=0, high=2, size=n_samples) + y = _enforce_estimator_tags_y(estimator, y) + + assert not hasattr(estimator, 'n_features_in_') + estimator.fit(X, y) + if hasattr(estimator, 'n_features_in_'): + assert estimator.n_features_in_ == X.shape[1] + else: + warnings.warn( + "As of scikit-learn 0.23, estimators should expose a " + "n_features_in_ attribute, unless the 'no_validation' tag is " + "True. This attribute should be equal to the number of features " + "passed to the fit method. " + "An error will be raised from version 0.25 when calling " + "check_estimator(). " + "See SLEP010: " + "https://scikit-learn-enhancement-proposals.readthedocs.io/en/latest/slep010/proposal.html", # noqa + FutureWarning + ) diff --git a/sklearn/utils/extmath.py b/sklearn/utils/extmath.py index b0c28897a8ef1..d11b307e7500e 100644 --- a/sklearn/utils/extmath.py +++ b/sklearn/utils/extmath.py @@ -184,10 +184,9 @@ def randomized_range_finder(A, size, n_iter, random_state : int, RandomState instance or None, optional (default=None) The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. + the data, i.e. getting the random vectors to initialize the algorithm. + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary `. Returns ------- @@ -296,10 +295,9 @@ def randomized_svd(M, n_components, n_oversamples=10, n_iter='auto', random_state : int, RandomState instance or None, optional (default=None) The seed of the pseudo random number generator to use when shuffling - the data. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number - generator; If None, the random number generator is the RandomState - instance used by `np.random`. + the data, i.e. getting the random vectors to initialize the algorithm. + Pass an int for reproducible results across multiple function calls. + See :term:`Glossary `. Notes ----- diff --git a/sklearn/utils/fixes.py b/sklearn/utils/fixes.py index 4456e9d271329..03e220eab29ae 100644 --- a/sklearn/utils/fixes.py +++ b/sklearn/utils/fixes.py @@ -173,16 +173,8 @@ def __getstate__(self): from numpy.ma import MaskedArray # noqa -# Fix for behavior inconsistency on numpy.equal for object dtypes. -# For numpy versions < 1.13, numpy.equal tests element-wise identity of objects -# instead of equality. This fix returns the mask of NaNs in an array of -# numerical or object values for all numpy versions. -if np_version < (1, 13): - def _object_dtype_isnan(X): - return np.frompyfunc(lambda x: x != x, 1, 1)(X).astype(bool) -else: - def _object_dtype_isnan(X): - return X != X +def _object_dtype_isnan(X): + return X != X # TODO: replace by copy=False, when only scipy > 1.1 is supported. diff --git a/sklearn/utils/graph_shortest_path.pyx b/sklearn/utils/graph_shortest_path.pyx index 30cbec1d5d471..7d2e74127f153 100644 --- a/sklearn/utils/graph_shortest_path.pyx +++ b/sklearn/utils/graph_shortest_path.pyx @@ -215,7 +215,7 @@ cdef np.ndarray dijkstra(dist_matrix, graph, &heap, nodes) else: #use the csr -> csc sparse matrix conversion to quickly get - # both directions of neigbors + # both directions of neighbors dist_matrix_T = dist_matrix.T.tocsr() distances2 = np.asarray(dist_matrix_T.data, diff --git a/sklearn/utils/linear_assignment_.py b/sklearn/utils/linear_assignment_.py deleted file mode 100644 index b396d90fb27bc..0000000000000 --- a/sklearn/utils/linear_assignment_.py +++ /dev/null @@ -1,284 +0,0 @@ -""" -Solve the unique lowest-cost assignment problem using the -Hungarian algorithm (also known as Munkres algorithm). - -""" -# Based on original code by Brain Clapper, adapted to NumPy by Gael Varoquaux. -# Heavily refactored by Lars Buitinck. - -# Copyright (c) 2008 Brian M. Clapper , Gael Varoquaux -# Author: Brian M. Clapper, Gael Varoquaux -# LICENSE: BSD - -import numpy as np -import warnings - - -# Deprecation warning for module -warnings.warn( - "The linear_assignment_ module is deprecated in 0.21 " - "and will be removed from 0.23. Use " - "scipy.optimize.linear_sum_assignment instead.", - FutureWarning) - - -def linear_assignment(X): - """Solve the linear assignment problem using the Hungarian algorithm. - - The problem is also known as maximum weight matching in bipartite graphs. - The method is also known as the Munkres or Kuhn-Munkres algorithm. - - Parameters - ---------- - X : array - The cost matrix of the bipartite graph - - Returns - ------- - indices : array - The pairs of (row, col) indices in the original array giving - the original ordering. - - References - ---------- - - 1. http://www.public.iastate.edu/~ddoty/HungarianAlgorithm.html - - 2. Harold W. Kuhn. The Hungarian Method for the assignment problem. - *Naval Research Logistics Quarterly*, 2:83-97, 1955. - - 3. Harold W. Kuhn. Variants of the Hungarian method for assignment - problems. *Naval Research Logistics Quarterly*, 3: 253-258, 1956. - - 4. Munkres, J. Algorithms for the Assignment and Transportation Problems. - *Journal of the Society of Industrial and Applied Mathematics*, - 5(1):32-38, March, 1957. - - 5. https://en.wikipedia.org/wiki/Hungarian_algorithm - """ - indices = _hungarian(X).tolist() - indices.sort() - # Re-force dtype to ints in case of empty list - indices = np.array(indices, dtype=int) - # Make sure the array is 2D with 2 columns. - # This is needed when dealing with an empty list - indices.shape = (-1, 2) - return indices - - -class _HungarianState: - """State of one execution of the Hungarian algorithm. - - Parameters - ---------- - cost_matrix : 2D matrix - The cost matrix. Does not have to be square. - """ - - def __init__(self, cost_matrix): - cost_matrix = np.atleast_2d(cost_matrix) - - # If there are more rows (n) than columns (m), then the algorithm - # will not be able to work correctly. Therefore, we - # transpose the cost function when needed. Just have to - # remember to swap the result columns back later. - transposed = (cost_matrix.shape[1] < cost_matrix.shape[0]) - if transposed: - self.C = (cost_matrix.T).copy() - else: - self.C = cost_matrix.copy() - self.transposed = transposed - - # At this point, m >= n. - n, m = self.C.shape - self.row_uncovered = np.ones(n, dtype=np.bool) - self.col_uncovered = np.ones(m, dtype=np.bool) - self.Z0_r = 0 - self.Z0_c = 0 - self.path = np.zeros((n + m, 2), dtype=int) - self.marked = np.zeros((n, m), dtype=int) - - def _clear_covers(self): - """Clear all covered matrix cells""" - self.row_uncovered[:] = True - self.col_uncovered[:] = True - - -def _hungarian(cost_matrix): - """The Hungarian algorithm. - - Calculate the Munkres solution to the classical assignment problem and - return the indices for the lowest-cost pairings. - - Parameters - ---------- - cost_matrix : 2D matrix - The cost matrix. Does not have to be square. - - Returns - ------- - indices : 2D array of indices - The pairs of (row, col) indices in the original array giving - the original ordering. - """ - warnings.warn( - "The linear_assignment function is deprecated in 0.21 " - "and will be removed from 0.23. Use " - "scipy.optimize.linear_sum_assignment instead.", - FutureWarning) - - state = _HungarianState(cost_matrix) - - # No need to bother with assignments if one of the dimensions - # of the cost matrix is zero-length. - step = None if 0 in cost_matrix.shape else _step1 - - while step is not None: - step = step(state) - - # Look for the starred columns - results = np.array(np.where(state.marked == 1)).T - - # We need to swap the columns because we originally - # did a transpose on the input cost matrix. - if state.transposed: - results = results[:, ::-1] - - return results - - -# Individual steps of the algorithm follow, as a state machine: they return -# the next step to be taken (function to be called), if any. - -def _step1(state): - """Steps 1 and 2 in the Wikipedia page.""" - - # Step1: For each row of the matrix, find the smallest element and - # subtract it from every element in its row. - state.C -= state.C.min(axis=1)[:, np.newaxis] - # Step2: Find a zero (Z) in the resulting matrix. If there is no - # starred zero in its row or column, star Z. Repeat for each element - # in the matrix. - for i, j in zip(*np.where(state.C == 0)): - if state.col_uncovered[j] and state.row_uncovered[i]: - state.marked[i, j] = 1 - state.col_uncovered[j] = False - state.row_uncovered[i] = False - - state._clear_covers() - return _step3 - - -def _step3(state): - """ - Cover each column containing a starred zero. If n columns are covered, - the starred zeros describe a complete set of unique assignments. - In this case, Go to DONE, otherwise, Go to Step 4. - """ - marked = (state.marked == 1) - state.col_uncovered[np.any(marked, axis=0)] = False - - if marked.sum() < state.C.shape[0]: - return _step4 - - -def _step4(state): - """ - Find a noncovered zero and prime it. If there is no starred zero - in the row containing this primed zero, Go to Step 5. Otherwise, - cover this row and uncover the column containing the starred - zero. Continue in this manner until there are no uncovered zeros - left. Save the smallest uncovered value and Go to Step 6. - """ - # We convert to int as numpy operations are faster on int - C = (state.C == 0).astype(np.int) - covered_C = C * state.row_uncovered[:, np.newaxis] - covered_C *= state.col_uncovered.astype(dtype=np.int, copy=False) - n = state.C.shape[0] - m = state.C.shape[1] - while True: - # Find an uncovered zero - row, col = np.unravel_index(np.argmax(covered_C), (n, m)) - if covered_C[row, col] == 0: - return _step6 - else: - state.marked[row, col] = 2 - # Find the first starred element in the row - star_col = np.argmax(state.marked[row] == 1) - if not state.marked[row, star_col] == 1: - # Could not find one - state.Z0_r = row - state.Z0_c = col - return _step5 - else: - col = star_col - state.row_uncovered[row] = False - state.col_uncovered[col] = True - covered_C[:, col] = C[:, col] * ( - state.row_uncovered.astype(dtype=np.int, copy=False)) - covered_C[row] = 0 - - -def _step5(state): - """ - Construct a series of alternating primed and starred zeros as follows. - Let Z0 represent the uncovered primed zero found in Step 4. - Let Z1 denote the starred zero in the column of Z0 (if any). - Let Z2 denote the primed zero in the row of Z1 (there will always be one). - Continue until the series terminates at a primed zero that has no starred - zero in its column. Unstar each starred zero of the series, star each - primed zero of the series, erase all primes and uncover every line in the - matrix. Return to Step 3 - """ - count = 0 - path = state.path - path[count, 0] = state.Z0_r - path[count, 1] = state.Z0_c - - while True: - # Find the first starred element in the col defined by - # the path. - row = np.argmax(state.marked[:, path[count, 1]] == 1) - if not state.marked[row, path[count, 1]] == 1: - # Could not find one - break - else: - count += 1 - path[count, 0] = row - path[count, 1] = path[count - 1, 1] - - # Find the first prime element in the row defined by the - # first path step - col = np.argmax(state.marked[path[count, 0]] == 2) - if state.marked[row, col] != 2: - col = -1 - count += 1 - path[count, 0] = path[count - 1, 0] - path[count, 1] = col - - # Convert paths - for i in range(count + 1): - if state.marked[path[i, 0], path[i, 1]] == 1: - state.marked[path[i, 0], path[i, 1]] = 0 - else: - state.marked[path[i, 0], path[i, 1]] = 1 - - state._clear_covers() - # Erase all prime markings - state.marked[state.marked == 2] = 0 - return _step3 - - -def _step6(state): - """ - Add the value found in Step 4 to every element of each covered row, - and subtract it from every element of each uncovered column. - Return to Step 4 without altering any stars, primes, or covered lines. - """ - # the smallest uncovered value in the matrix - if np.any(state.row_uncovered) and np.any(state.col_uncovered): - minval = np.min(state.C[state.row_uncovered], axis=0) - minval = np.min(minval[state.col_uncovered]) - state.C[np.logical_not(state.row_uncovered)] += minval - state.C[:, state.col_uncovered] -= minval - return _step4 diff --git a/sklearn/utils/optimize.py b/sklearn/utils/optimize.py index f83d2ffc375ae..b1767cea59866 100644 --- a/sklearn/utils/optimize.py +++ b/sklearn/utils/optimize.py @@ -182,7 +182,7 @@ def _newton_cg(grad_hess, func, grad, x0, args=(), tol=1e-4, fgrad, fhess_p = grad_hess(xk, *args) absgrad = np.abs(fgrad) - if np.max(absgrad) < tol: + if np.max(absgrad) <= tol: break maggrad = np.sum(absgrad) @@ -213,7 +213,8 @@ def _newton_cg(grad_hess, func, grad, x0, args=(), tol=1e-4, return xk, k -def _check_optimize_result(solver, result, max_iter=None): +def _check_optimize_result(solver, result, max_iter=None, + extra_warning_msg=None): """Check the OptimizeResult for successful convergence Parameters @@ -233,10 +234,16 @@ def _check_optimize_result(solver, result, max_iter=None): # handle both scipy and scikit-learn solver names if solver == "lbfgs": if result.status != 0: - warnings.warn("{} failed to converge (status={}): {}. " - "Increase the number of iterations." - .format(solver, result.status, result.message), - ConvergenceWarning, stacklevel=2) + warning_msg = ( + "{} failed to converge (status={}):\n{}.\n\n" + "Increase the number of iterations (max_iter) " + "or scale the data as shown in:\n" + " https://scikit-learn.org/stable/modules/" + "preprocessing.html" + ).format(solver, result.status, result.message.decode("latin1")) + if extra_warning_msg is not None: + warning_msg += "\n" + extra_warning_msg + warnings.warn(warning_msg, ConvergenceWarning, stacklevel=2) if max_iter is not None: # In scipy <= 1.0.0, nit may exceed maxiter for lbfgs. # See https://github.com/scipy/scipy/issues/7854 diff --git a/sklearn/utils/random.py b/sklearn/utils/random.py index d59d578ff1a1d..524c90406e6eb 100644 --- a/sklearn/utils/random.py +++ b/sklearn/utils/random.py @@ -36,11 +36,9 @@ def _random_choice_csc(n_samples, classes, class_probability=None, Optional (default=None). Class distribution of each column. If None the uniform distribution is assumed. - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. + random_state : int, RandomState instance, default=None + Controls the randomness of the sampled classes. + See :term:`Glossary `. Returns ------- diff --git a/sklearn/utils/sparsefuncs_fast.pyx b/sklearn/utils/sparsefuncs_fast.pyx index 7fdd9168a50e3..35abccf284088 100644 --- a/sklearn/utils/sparsefuncs_fast.pyx +++ b/sklearn/utils/sparsefuncs_fast.pyx @@ -25,6 +25,7 @@ ctypedef fused integral: ctypedef np.float64_t DOUBLE + def csr_row_norms(X): """L2 norm of each row in CSR matrix X.""" if X.dtype not in [np.float32, np.float64]: @@ -38,19 +39,18 @@ def _csr_row_norms(np.ndarray[floating, ndim=1, mode="c"] X_data, np.ndarray[integral, ndim=1, mode="c"] X_indptr): cdef: unsigned long long n_samples = shape[0] - unsigned long long n_features = shape[1] - np.ndarray[DOUBLE, ndim=1, mode="c"] norms - - np.npy_intp i, j + unsigned long long i + integral j double sum_ - norms = np.zeros(n_samples, dtype=np.float64) + norms = np.empty(n_samples, dtype=X_data.dtype) + cdef floating[::1] norms_view = norms for i in range(n_samples): sum_ = 0.0 for j in range(X_indptr[i], X_indptr[i + 1]): sum_ += X_data[j] * X_data[j] - norms[i] = sum_ + norms_view[i] = sum_ return norms @@ -334,23 +334,26 @@ def _incr_mean_variance_axis0(np.ndarray[floating, ndim=1] X_data, # Next passes for i in range(n_features): - updated_n[i] = last_n[i] + new_n[i] - last_over_new_n[i] = last_n[i] / new_n[i] - - # Unnormalized stats - for i in range(n_features): - last_mean[i] *= last_n[i] - last_var[i] *= last_n[i] - new_mean[i] *= new_n[i] - new_var[i] *= new_n[i] - - # Update stats - for i in range(n_features): - updated_var[i] = (last_var[i] + new_var[i] + - last_over_new_n[i] / updated_n[i] * - (last_mean[i] / last_over_new_n[i] - new_mean[i])**2) - updated_mean[i] = (last_mean[i] + new_mean[i]) / updated_n[i] - updated_var[i] /= updated_n[i] + if new_n[i] > 0: + updated_n[i] = last_n[i] + new_n[i] + last_over_new_n[i] = dtype(last_n[i]) / dtype(new_n[i]) + # Unnormalized stats + last_mean[i] *= last_n[i] + last_var[i] *= last_n[i] + new_mean[i] *= new_n[i] + new_var[i] *= new_n[i] + # Update stats + updated_var[i] = ( + last_var[i] + new_var[i] + + last_over_new_n[i] / updated_n[i] * + (last_mean[i] / last_over_new_n[i] - new_mean[i])**2 + ) + updated_mean[i] = (last_mean[i] + new_mean[i]) / updated_n[i] + updated_var[i] /= updated_n[i] + else: + updated_var[i] = last_var[i] + updated_mean[i] = last_mean[i] + updated_n[i] = last_n[i] return updated_mean, updated_var, updated_n diff --git a/sklearn/utils/tests/test_class_weight.py b/sklearn/utils/tests/test_class_weight.py index 31e396ce6b2f5..487da5b431be0 100644 --- a/sklearn/utils/tests/test_class_weight.py +++ b/sklearn/utils/tests/test_class_weight.py @@ -1,15 +1,13 @@ import numpy as np +import pytest -from sklearn.linear_model import LogisticRegression from sklearn.datasets import make_blobs +from sklearn.linear_model import LogisticRegression from sklearn.utils.class_weight import compute_class_weight from sklearn.utils.class_weight import compute_sample_weight - from sklearn.utils._testing import assert_array_almost_equal from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import assert_raises -from sklearn.utils._testing import assert_raise_message def test_compute_class_weight(): @@ -28,17 +26,19 @@ def test_compute_class_weight_not_present(): # Raise error when y does not contain all class labels classes = np.arange(4) y = np.asarray([0, 0, 0, 1, 1, 2]) - assert_raises(ValueError, compute_class_weight, "balanced", classes, y) + with pytest.raises(ValueError): + compute_class_weight("balanced", classes, y) # Fix exception in error message formatting when missing label is a string # https://github.com/scikit-learn/scikit-learn/issues/8312 - assert_raise_message(ValueError, - 'Class label label_not_present not present', - compute_class_weight, - {'label_not_present': 1.}, classes, y) + with pytest.raises(ValueError, + match="Class label label_not_present not present"): + compute_class_weight({"label_not_present": 1.}, classes, y) # Raise error when y has items not in classes classes = np.arange(2) - assert_raises(ValueError, compute_class_weight, "balanced", classes, y) - assert_raises(ValueError, compute_class_weight, {0: 1., 1: 2.}, classes, y) + with pytest.raises(ValueError): + compute_class_weight("balanced", classes, y) + with pytest.raises(ValueError): + compute_class_weight({0: 1., 1: 2.}, classes, y) def test_compute_class_weight_dict(): @@ -55,12 +55,13 @@ def test_compute_class_weight_dict(): # should get raised msg = 'Class label 4 not present.' class_weights = {0: 1.0, 1: 2.0, 2: 3.0, 4: 1.5} - assert_raise_message(ValueError, msg, compute_class_weight, class_weights, - classes, y) + with pytest.raises(ValueError, match=msg): + compute_class_weight(class_weights, classes, y) + msg = 'Class label -1 not present.' class_weights = {-1: 5.0, 0: 1.0, 1: 2.0, 2: 3.0} - assert_raise_message(ValueError, msg, compute_class_weight, class_weights, - classes, y) + with pytest.raises(ValueError, match=msg): + compute_class_weight(class_weights, classes, y) def test_compute_class_weight_invariance(): @@ -232,20 +233,27 @@ def test_compute_sample_weight_errors(): # Invalid preset string y = np.asarray([1, 1, 1, 2, 2, 2]) y_ = np.asarray([[1, 0], [1, 0], [1, 0], [2, 1], [2, 1], [2, 1]]) - assert_raises(ValueError, compute_sample_weight, "ni", y) - assert_raises(ValueError, compute_sample_weight, "ni", y, range(4)) - assert_raises(ValueError, compute_sample_weight, "ni", y_) - assert_raises(ValueError, compute_sample_weight, "ni", y_, range(4)) + + with pytest.raises(ValueError): + compute_sample_weight("ni", y) + with pytest.raises(ValueError): + compute_sample_weight("ni", y, range(4)) + with pytest.raises(ValueError): + compute_sample_weight("ni", y_) + with pytest.raises(ValueError): + compute_sample_weight("ni", y_, range(4)) # Not "balanced" for subsample - assert_raises(ValueError, - compute_sample_weight, {1: 2, 2: 1}, y, range(4)) + with pytest.raises(ValueError): + compute_sample_weight({1: 2, 2: 1}, y, range(4)) # Not a list or preset for multi-output - assert_raises(ValueError, compute_sample_weight, {1: 2, 2: 1}, y_) + with pytest.raises(ValueError): + compute_sample_weight({1: 2, 2: 1}, y_) # Incorrect length list for multi-output - assert_raises(ValueError, compute_sample_weight, [{1: 2, 2: 1}], y_) + with pytest.raises(ValueError): + compute_sample_weight([{1: 2, 2: 1}], y_) def test_compute_sample_weight_more_than_32(): diff --git a/sklearn/utils/tests/test_cython_blas.py b/sklearn/utils/tests/test_cython_blas.py index d55b274a6f0db..eb33e9455a563 100644 --- a/sklearn/utils/tests/test_cython_blas.py +++ b/sklearn/utils/tests/test_cython_blas.py @@ -17,10 +17,15 @@ from sklearn.utils._cython_blas import RowMajor, ColMajor from sklearn.utils._cython_blas import Trans, NoTrans -cython = pytest.importorskip("cython") + +def _numpy_to_cython(dtype): + cython = pytest.importorskip("cython") + if dtype == np.float32: + return cython.float + elif dtype == np.float64: + return cython.double -NUMPY_TO_CYTHON = {np.float32: cython.float, np.float64: cython.double} RTOL = {np.float32: 1e-6, np.float64: 1e-12} ORDER = {RowMajor: 'C', ColMajor: 'F'} @@ -31,7 +36,7 @@ def _no_op(x): @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_dot(dtype): - dot = _dot_memview[NUMPY_TO_CYTHON[dtype]] + dot = _dot_memview[_numpy_to_cython(dtype)] rng = np.random.RandomState(0) x = rng.random_sample(10).astype(dtype, copy=False) @@ -45,7 +50,7 @@ def test_dot(dtype): @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_asum(dtype): - asum = _asum_memview[NUMPY_TO_CYTHON[dtype]] + asum = _asum_memview[_numpy_to_cython(dtype)] rng = np.random.RandomState(0) x = rng.random_sample(10).astype(dtype, copy=False) @@ -58,7 +63,7 @@ def test_asum(dtype): @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_axpy(dtype): - axpy = _axpy_memview[NUMPY_TO_CYTHON[dtype]] + axpy = _axpy_memview[_numpy_to_cython(dtype)] rng = np.random.RandomState(0) x = rng.random_sample(10).astype(dtype, copy=False) @@ -73,7 +78,7 @@ def test_axpy(dtype): @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_nrm2(dtype): - nrm2 = _nrm2_memview[NUMPY_TO_CYTHON[dtype]] + nrm2 = _nrm2_memview[_numpy_to_cython(dtype)] rng = np.random.RandomState(0) x = rng.random_sample(10).astype(dtype, copy=False) @@ -86,7 +91,7 @@ def test_nrm2(dtype): @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_copy(dtype): - copy = _copy_memview[NUMPY_TO_CYTHON[dtype]] + copy = _copy_memview[_numpy_to_cython(dtype)] rng = np.random.RandomState(0) x = rng.random_sample(10).astype(dtype, copy=False) @@ -100,7 +105,7 @@ def test_copy(dtype): @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_scal(dtype): - scal = _scal_memview[NUMPY_TO_CYTHON[dtype]] + scal = _scal_memview[_numpy_to_cython(dtype)] rng = np.random.RandomState(0) x = rng.random_sample(10).astype(dtype, copy=False) @@ -114,7 +119,7 @@ def test_scal(dtype): @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_rotg(dtype): - rotg = _rotg_memview[NUMPY_TO_CYTHON[dtype]] + rotg = _rotg_memview[_numpy_to_cython(dtype)] rng = np.random.RandomState(0) a = dtype(rng.randn()) @@ -139,7 +144,7 @@ def expected_rotg(a, b): @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_rot(dtype): - rot = _rot_memview[NUMPY_TO_CYTHON[dtype]] + rot = _rot_memview[_numpy_to_cython(dtype)] rng = np.random.RandomState(0) x = rng.random_sample(10).astype(dtype, copy=False) @@ -163,7 +168,7 @@ def test_rot(dtype): @pytest.mark.parametrize("order", [RowMajor, ColMajor], ids=["RowMajor", "ColMajor"]) def test_gemv(dtype, opA, transA, order): - gemv = _gemv_memview[NUMPY_TO_CYTHON[dtype]] + gemv = _gemv_memview[_numpy_to_cython(dtype)] rng = np.random.RandomState(0) A = np.asarray(opA(rng.random_sample((20, 10)).astype(dtype, copy=False)), @@ -182,7 +187,7 @@ def test_gemv(dtype, opA, transA, order): @pytest.mark.parametrize("order", [RowMajor, ColMajor], ids=["RowMajor", "ColMajor"]) def test_ger(dtype, order): - ger = _ger_memview[NUMPY_TO_CYTHON[dtype]] + ger = _ger_memview[_numpy_to_cython(dtype)] rng = np.random.RandomState(0) x = rng.random_sample(10).astype(dtype, copy=False) @@ -207,7 +212,7 @@ def test_ger(dtype, order): @pytest.mark.parametrize("order", [RowMajor, ColMajor], ids=["RowMajor", "ColMajor"]) def test_gemm(dtype, opA, transA, opB, transB, order): - gemm = _gemm_memview[NUMPY_TO_CYTHON[dtype]] + gemm = _gemm_memview[_numpy_to_cython(dtype)] rng = np.random.RandomState(0) A = np.asarray(opA(rng.random_sample((30, 10)).astype(dtype, copy=False)), diff --git a/sklearn/utils/tests/test_deprecated_utils.py b/sklearn/utils/tests/test_deprecated_utils.py index da41e7e44ddb3..c23c72866666b 100644 --- a/sklearn/utils/tests/test_deprecated_utils.py +++ b/sklearn/utils/tests/test_deprecated_utils.py @@ -1,7 +1,10 @@ import pytest +import types import numpy as np +import warnings from sklearn.dummy import DummyClassifier +from sklearn.utils import all_estimators from sklearn.utils.estimator_checks import choose_check_classifiers_labels from sklearn.utils.estimator_checks import NotAnArray from sklearn.utils.estimator_checks import enforce_estimator_tags_y @@ -71,9 +74,6 @@ def func(x): def grad(x): return A.T.dot(A.dot(x)) - def hess(x, p): - return p.dot(A.T.dot(A.dot(x.all()))) - def grad_hess(x): return grad(x), lambda x: A.T.dot(A.dot(x)) @@ -94,3 +94,35 @@ def test_safe_indexing(): with pytest.warns(FutureWarning, match="removed in version 0.24"): safe_indexing([1, 2], 0) + + +# TODO: remove in 0.24 +def test_partial_dependence_no_shadowing(): + # Non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/15842 + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=FutureWarning) + from sklearn.inspection.partial_dependence import partial_dependence as _ # noqa + + # Calling all_estimators() also triggers a recursive import of all + # submodules, including deprecated ones. + all_estimators() + + from sklearn.inspection import partial_dependence + assert isinstance(partial_dependence, types.FunctionType) + + +# TODO: remove in 0.24 +def test_dict_learning_no_shadowing(): + # Non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/15842 + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=FutureWarning) + from sklearn.decomposition.dict_learning import dict_learning as _ # noqa + + # Calling all_estimators() also triggers a recursive import of all + # submodules, including deprecated ones. + all_estimators() + + from sklearn.decomposition import dict_learning + assert isinstance(dict_learning, types.FunctionType) diff --git a/sklearn/utils/tests/test_estimator_checks.py b/sklearn/utils/tests/test_estimator_checks.py index f0c014829483f..748666884e60e 100644 --- a/sklearn/utils/tests/test_estimator_checks.py +++ b/sklearn/utils/tests/test_estimator_checks.py @@ -10,9 +10,9 @@ from sklearn.base import BaseEstimator, ClassifierMixin from sklearn.utils import deprecated from sklearn.utils._testing import (assert_raises_regex, - ignore_warnings, - assert_warns, assert_raises, - SkipTest) + ignore_warnings, + assert_warns, assert_raises, + SkipTest) from sklearn.utils.estimator_checks import check_estimator, _NotAnArray from sklearn.utils.estimator_checks \ import check_class_weight_balanced_linear_classifier @@ -21,6 +21,8 @@ from sklearn.utils.estimator_checks import check_estimators_unfitted from sklearn.utils.estimator_checks import check_fit_score_takes_y from sklearn.utils.estimator_checks import check_no_attributes_set_in_init +from sklearn.utils.estimator_checks import check_classifier_data_not_an_array +from sklearn.utils.estimator_checks import check_regressor_data_not_an_array from sklearn.utils.validation import check_is_fitted from sklearn.utils.estimator_checks import check_outlier_corruption from sklearn.utils.fixes import _parse_version @@ -34,6 +36,7 @@ from sklearn.neighbors import KNeighborsRegressor from sklearn.tree import DecisionTreeClassifier from sklearn.utils.validation import check_X_y, check_array +from sklearn.utils import all_estimators class CorrectNotFittedError(ValueError): @@ -57,7 +60,7 @@ def __init__(self, key=0): self.key = key def fit(self, X, y=None): - X, y = check_X_y(X, y) + X, y = self._validate_data(X, y) return self def predict(self, X): @@ -72,7 +75,7 @@ def __init__(self, acceptable_key=0): def fit(self, X, y=None): self.wrong_attribute = 0 - X, y = check_X_y(X, y) + X, y = self._validate_data(X, y) return self @@ -82,14 +85,14 @@ def __init__(self, wrong_attribute=0): def fit(self, X, y=None): self.wrong_attribute = 1 - X, y = check_X_y(X, y) + X, y = self._validate_data(X, y) return self class ChangesUnderscoreAttribute(BaseEstimator): def fit(self, X, y=None): self._good_attribute = 1 - X, y = check_X_y(X, y) + X, y = self._validate_data(X, y) return self @@ -106,7 +109,7 @@ def set_params(self, **kwargs): return super().set_params(**kwargs) def fit(self, X, y=None): - X, y = check_X_y(X, y) + X, y = self._validate_data(X, y) return self @@ -123,7 +126,7 @@ def set_params(self, **kwargs): return super().set_params(**kwargs) def fit(self, X, y=None): - X, y = check_X_y(X, y) + X, y = self._validate_data(X, y) return self @@ -142,19 +145,19 @@ def set_params(self, **kwargs): return super().set_params(**kwargs) def fit(self, X, y=None): - X, y = check_X_y(X, y) + X, y = self._validate_data(X, y) return self class NoCheckinPredict(BaseBadClassifier): def fit(self, X, y): - X, y = check_X_y(X, y) + X, y = self._validate_data(X, y) return self class NoSparseClassifier(BaseBadClassifier): def fit(self, X, y): - X, y = check_X_y(X, y, accept_sparse=['csr', 'csc']) + X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc']) if sp.issparse(X): raise ValueError("Nonsensical Error") return self @@ -166,7 +169,7 @@ def predict(self, X): class CorrectNotFittedErrorClassifier(BaseBadClassifier): def fit(self, X, y): - X, y = check_X_y(X, y) + X, y = self._validate_data(X, y) self.coef_ = np.ones(X.shape[1]) return self @@ -179,10 +182,11 @@ def predict(self, X): class NoSampleWeightPandasSeriesType(BaseEstimator): def fit(self, X, y, sample_weight=None): # Convert data - X, y = check_X_y(X, y, - accept_sparse=("csr", "csc"), - multi_output=True, - y_numeric=True) + X, y = self._validate_data( + X, y, + accept_sparse=("csr", "csc"), + multi_output=True, + y_numeric=True) # Function is only called after we verify that pandas is installed from pandas import Series if isinstance(sample_weight, Series): @@ -219,7 +223,7 @@ def fit(self, X, y): class BadTransformerWithoutMixin(BaseEstimator): def fit(self, X, y=None): - X = check_array(X) + X = self._validate_data(X) return self def transform(self, X): @@ -230,10 +234,11 @@ def transform(self, X): class NotInvariantPredict(BaseEstimator): def fit(self, X, y): # Convert data - X, y = check_X_y(X, y, - accept_sparse=("csr", "csc"), - multi_output=True, - y_numeric=True) + X, y = self._validate_data( + X, y, + accept_sparse=("csr", "csc"), + multi_output=True, + y_numeric=True) return self def predict(self, X): @@ -246,27 +251,27 @@ def predict(self, X): class LargeSparseNotSupportedClassifier(BaseEstimator): def fit(self, X, y): - X, y = check_X_y(X, y, - accept_sparse=("csr", "csc", "coo"), - accept_large_sparse=True, - multi_output=True, - y_numeric=True) + X, y = self._validate_data( + X, y, + accept_sparse=("csr", "csc", "coo"), + accept_large_sparse=True, + multi_output=True, + y_numeric=True) if sp.issparse(X): if X.getformat() == "coo": if X.row.dtype == "int64" or X.col.dtype == "int64": raise ValueError( "Estimator doesn't support 64-bit indices") elif X.getformat() in ["csc", "csr"]: - if X.indices.dtype == "int64" or X.indptr.dtype == "int64": - raise ValueError( - "Estimator doesn't support 64-bit indices") + assert "int64" not in (X.indices.dtype, X.indptr.dtype),\ + "Estimator doesn't support 64-bit indices" return self class SparseTransformer(BaseEstimator): def fit(self, X, y=None): - self.X_shape_ = check_array(X).shape + self.X_shape_ = self._validate_data(X).shape return self def fit_transform(self, X, y=None): @@ -279,6 +284,27 @@ def transform(self, X): return sp.csr_matrix(X) +class EstimatorInconsistentForPandas(BaseEstimator): + def fit(self, X, y): + try: + from pandas import DataFrame + if isinstance(X, DataFrame): + self.value_ = X.iloc[0, 0] + else: + X = check_array(X) + self.value_ = X[1, 0] + return self + + except ImportError: + X = check_array(X) + self.value_ = X[1, 0] + return self + + def predict(self, X): + X = check_array(X) + return np.array([self.value_] * X.shape[0]) + + class UntaggedBinaryClassifier(DecisionTreeClassifier): # Toy classifier that only supports binary classification, will fail tests. def fit(self, X, y, sample_weight=None): @@ -297,7 +323,7 @@ def _more_tags(self): class RequiresPositiveYRegressor(LinearRegression): def fit(self, X, y): - X, y = check_X_y(X, y, multi_output=True) + X, y = self._validate_data(X, y, multi_output=True) if (y <= 0).any(): raise ValueError('negative y values not supported!') return super().fit(X, y) @@ -335,7 +361,7 @@ def test_check_estimator(): # not a complete test of all checks, which are very extensive. # check that we have a set_params and can clone - msg = "it does not implement a 'get_params' methods" + msg = "it does not implement a 'get_params' method" assert_raises_regex(TypeError, msg, check_estimator, object) assert_raises_regex(TypeError, msg, check_estimator, object()) # check that values returned by get_params match set_params @@ -536,6 +562,22 @@ def test_check_estimator_pairwise(): check_estimator(est) +def test_check_classifier_data_not_an_array(): + assert_raises_regex(AssertionError, + 'Not equal to tolerance', + check_classifier_data_not_an_array, + 'estimator_name', + EstimatorInconsistentForPandas()) + + +def test_check_regressor_data_not_an_array(): + assert_raises_regex(AssertionError, + 'Not equal to tolerance', + check_regressor_data_not_an_array, + 'estimator_name', + EstimatorInconsistentForPandas()) + + def test_check_estimator_required_parameters_skip(): class MyEstimator(BaseEstimator): _required_parameters = ["special_parameter"] @@ -572,6 +614,14 @@ def test_check_class_weight_balanced_linear_classifier(): BadBalancedWeightsClassifier) +def test_all_estimators_all_public(): + # all_estimator should not fail when pytest is not installed and return + # only public estimators + estimators = all_estimators() + for est in estimators: + assert not est.__class__.__name__.startswith("_") + + if __name__ == '__main__': # This module is run as a script to check that we have no dependency on # pytest for estimator checks. diff --git a/sklearn/utils/tests/test_extmath.py b/sklearn/utils/tests/test_extmath.py index fdca303e15d8b..2abcbfa3c74e7 100644 --- a/sklearn/utils/tests/test_extmath.py +++ b/sklearn/utils/tests/test_extmath.py @@ -545,15 +545,8 @@ def naive_mean_variance_update(x, last_mean, last_variance, A1 = np.full((n_samples // 2, n_features), x2, dtype=np.float64) A = np.vstack((A0, A1)) - # Older versions of numpy have different precision - # In some old version, np.var is not stable - if np.abs(np_var(A) - two_pass_var(A)).max() < 1e-6: - stable_var = np_var - else: - stable_var = two_pass_var - # Naive one pass var: >tol (=1063) - assert np.abs(stable_var(A) - one_pass_var(A)).max() > tol + assert np.abs(np_var(A) - one_pass_var(A)).max() > tol # Starting point for online algorithms: after A0 @@ -565,7 +558,7 @@ def naive_mean_variance_update(x, last_mean, last_variance, assert n == A.shape[0] # the mean is also slightly unstable assert np.abs(A.mean(axis=0) - mean).max() > 1e-6 - assert np.abs(stable_var(A) - var).max() > tol + assert np.abs(np_var(A) - var).max() > tol # Robust implementation: np.abs(stable_var(A) - var).max() + assert tol > np.abs(np_var(A) - var).max() def test_incremental_variance_ddof(): diff --git a/sklearn/utils/tests/test_linear_assignment.py b/sklearn/utils/tests/test_linear_assignment.py deleted file mode 100644 index 2f9399e68606c..0000000000000 --- a/sklearn/utils/tests/test_linear_assignment.py +++ /dev/null @@ -1,64 +0,0 @@ -# Author: Brian M. Clapper, G Varoquaux -# License: BSD - -# TODO #0.23: Remove this test module as the methods being tested -# have been replaced by SciPy methods - -import numpy as np -import pytest - - -@pytest.mark.filterwarnings( - "ignore::FutureWarning") -def test_hungarian(): - from sklearn.utils.linear_assignment_ import _hungarian - matrices = [ - # Square - ([[400, 150, 400], - [400, 450, 600], - [300, 225, 300]], - 850 # expected cost - ), - - # Rectangular variant - ([[400, 150, 400, 1], - [400, 450, 600, 2], - [300, 225, 300, 3]], - 452 # expected cost - ), - - # Square - ([[10, 10, 8], - [9, 8, 1], - [9, 7, 4]], - 18 - ), - - # Rectangular variant - ([[10, 10, 8, 11], - [9, 8, 1, 1], - [9, 7, 4, 10]], - 15 - ), - - # n == 2, m == 0 matrix - ([[], []], - 0 - ), - ] - - for cost_matrix, expected_total in matrices: - cost_matrix = np.array(cost_matrix) - indexes = _hungarian(cost_matrix) - total_cost = 0 - for r, c in indexes: - x = cost_matrix[r, c] - total_cost += x - assert expected_total == total_cost - - indexes = _hungarian(cost_matrix.T) - total_cost = 0 - for c, r in indexes: - x = cost_matrix[r, c] - total_cost += x - assert expected_total == total_cost diff --git a/sklearn/utils/tests/test_multiclass.py b/sklearn/utils/tests/test_multiclass.py index 022252d0c4836..4cb7368d34c03 100644 --- a/sklearn/utils/tests/test_multiclass.py +++ b/sklearn/utils/tests/test_multiclass.py @@ -3,6 +3,7 @@ import scipy.sparse as sp from itertools import product import pytest +from distutils.version import LooseVersion from scipy.sparse import issparse from scipy.sparse import csc_matrix @@ -13,8 +14,6 @@ from sklearn.utils._testing import assert_array_equal from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_raises -from sklearn.utils._testing import assert_raises_regex from sklearn.utils._testing import assert_allclose from sklearn.utils.estimator_checks import _NotAnArray @@ -153,7 +152,8 @@ def test_unique_labels(): # Empty iterable - assert_raises(ValueError, unique_labels) + with pytest.raises(ValueError): + unique_labels() # Multiclass problem assert_array_equal(unique_labels(range(10)), np.arange(10)) @@ -177,8 +177,11 @@ def test_unique_labels(): np.arange(3)) # Border line case with binary indicator matrix - assert_raises(ValueError, unique_labels, [4, 0, 2], np.ones((5, 5))) - assert_raises(ValueError, unique_labels, np.ones((5, 4)), np.ones((5, 5))) + with pytest.raises(ValueError): + unique_labels([4, 0, 2], np.ones((5, 5))) + with pytest.raises(ValueError): + unique_labels(np.ones((5, 4)), np.ones((5, 5))) + assert_array_equal(unique_labels(np.ones((4, 5)), np.ones((5, 5))), np.arange(5)) @@ -193,12 +196,14 @@ def test_unique_labels_non_specific(): # We don't support those format at the moment for example in NON_ARRAY_LIKE_EXAMPLES: - assert_raises(ValueError, unique_labels, example) + with pytest.raises(ValueError): + unique_labels(example) for y_type in ["unknown", "continuous", 'continuous-multioutput', 'multiclass-multioutput']: for example in EXAMPLES[y_type]: - assert_raises(ValueError, unique_labels, example) + with pytest.raises(ValueError): + unique_labels(example) def test_unique_labels_mixed_types(): @@ -208,13 +213,22 @@ def test_unique_labels_mixed_types(): EXAMPLES["binary"]) for y_multilabel, y_multiclass in mix_clf_format: - assert_raises(ValueError, unique_labels, y_multiclass, y_multilabel) - assert_raises(ValueError, unique_labels, y_multilabel, y_multiclass) + with pytest.raises(ValueError): + unique_labels(y_multiclass, y_multilabel) + with pytest.raises(ValueError): + unique_labels(y_multilabel, y_multiclass) + + with pytest.raises(ValueError): + unique_labels([[1, 2]], [["a", "d"]]) + + with pytest.raises(ValueError): + unique_labels(["1", 2]) - assert_raises(ValueError, unique_labels, [[1, 2]], [["a", "d"]]) - assert_raises(ValueError, unique_labels, ["1", 2]) - assert_raises(ValueError, unique_labels, [["1", 2], [1, 3]]) - assert_raises(ValueError, unique_labels, [["1", "2"], [2, 3]]) + with pytest.raises(ValueError): + unique_labels([["1", 2], [1, 3]]) + + with pytest.raises(ValueError): + unique_labels([["1", "2"], [2, 3]]) def test_is_multilabel(): @@ -262,8 +276,8 @@ def test_check_classification_targets(): if y_type in ["unknown", "continuous", 'continuous-multioutput']: for example in EXAMPLES[y_type]: msg = 'Unknown label type: ' - assert_raises_regex(ValueError, msg, - check_classification_targets, example) + with pytest.raises(ValueError, match=msg): + check_classification_targets(example) else: for example in EXAMPLES[y_type]: check_classification_targets(example) @@ -279,19 +293,26 @@ def test_type_of_target(): for example in NON_ARRAY_LIKE_EXAMPLES: msg_regex = r'Expected array-like \(array or non-string sequence\).*' - assert_raises_regex(ValueError, msg_regex, type_of_target, example) + with pytest.raises(ValueError, match=msg_regex): + type_of_target(example) for example in MULTILABEL_SEQUENCES: msg = ('You appear to be using a legacy multi-label data ' 'representation. Sequence of sequences are no longer supported;' ' use a binary array or sparse matrix instead.') - assert_raises_regex(ValueError, msg, type_of_target, example) + with pytest.raises(ValueError, match=msg): + type_of_target(example) def test_type_of_target_pandas_sparse(): pd = pytest.importorskip("pandas") - y = pd.SparseArray([1, np.nan, np.nan, 1, np.nan]) + if LooseVersion(pd.__version__) >= '0.25': + pd_sparse_array = pd.arrays.SparseArray + else: + pd_sparse_array = pd.SparseArray + + y = pd_sparse_array([1, np.nan, np.nan, 1, np.nan]) msg = "y cannot be class 'SparseSeries' or 'SparseArray'" with pytest.raises(ValueError, match=msg): type_of_target(y) diff --git a/sklearn/utils/tests/test_pprint.py b/sklearn/utils/tests/test_pprint.py index 556d57e9f8dfa..146ccf781ae8a 100644 --- a/sklearn/utils/tests/test_pprint.py +++ b/sklearn/utils/tests/test_pprint.py @@ -371,6 +371,7 @@ def test_gridsearch_pipeline(): 'function chi2 at some_address>', repr_) assert repr_ == expected + def test_n_max_elements_to_show(): n_max_elements_to_show = 30 @@ -397,7 +398,7 @@ def test_n_max_elements_to_show(): 27: 27, 28: 28, 29: 29})""" expected = expected[1:] # remove first \n - assert pp.pformat(vectorizer) == expected + assert pp.pformat(vectorizer) == expected # Now with ellipsis vocabulary = {i: i for i in range(n_max_elements_to_show + 1)} @@ -417,7 +418,7 @@ def test_n_max_elements_to_show(): 27: 27, 28: 28, 29: 29, ...})""" expected = expected[1:] # remove first \n - assert pp.pformat(vectorizer) == expected + assert pp.pformat(vectorizer) == expected # Also test with lists param_grid = {'C': list(range(n_max_elements_to_show))} @@ -437,7 +438,7 @@ def test_n_max_elements_to_show(): scoring=None, verbose=0)""" expected = expected[1:] # remove first \n - assert pp.pformat(gs) == expected + assert pp.pformat(gs) == expected # Now with ellipsis param_grid = {'C': list(range(n_max_elements_to_show + 1))} @@ -457,7 +458,7 @@ def test_n_max_elements_to_show(): scoring=None, verbose=0)""" expected = expected[1:] # remove first \n - assert pp.pformat(gs) == expected + assert pp.pformat(gs) == expected def test_bruteforce_ellipsis(): @@ -533,6 +534,7 @@ def test_bruteforce_ellipsis(): expected = expected[1:] # remove first \n assert expected == lr.__repr__(N_CHAR_MAX=n_nonblank - 2) + def test_builtin_prettyprinter(): # non regression test than ensures we can still use the builtin # PrettyPrinter class for estimators (as done e.g. by joblib). diff --git a/sklearn/utils/tests/test_random.py b/sklearn/utils/tests/test_random.py index 2ed7dbce128e9..7d2437471aabb 100644 --- a/sklearn/utils/tests/test_random.py +++ b/sklearn/utils/tests/test_random.py @@ -1,18 +1,19 @@ import numpy as np +import pytest import scipy.sparse as sp from numpy.testing import assert_array_almost_equal from sklearn.utils.fixes import comb from sklearn.utils.random import _random_choice_csc, sample_without_replacement from sklearn.utils._random import _our_rand_r_py -from sklearn.utils._testing import assert_raises ############################################################################### # test custom sampling without replacement algorithm ############################################################################### def test_invalid_sample_without_replacement_algorithm(): - assert_raises(ValueError, sample_without_replacement, 5, 4, "unknown") + with pytest.raises(ValueError): + sample_without_replacement(5, 4, "unknown") def test_sample_without_replacement_algorithms(): @@ -33,8 +34,10 @@ def sample_without_replacement_method(n_population, n_samples, def check_edge_case_of_sample_int(sample_without_replacement): # n_population < n_sample - assert_raises(ValueError, sample_without_replacement, 0, 1) - assert_raises(ValueError, sample_without_replacement, 1, 2) + with pytest.raises(ValueError): + sample_without_replacement(0, 1) + with pytest.raises(ValueError): + sample_without_replacement(1, 2) # n_population == n_samples assert sample_without_replacement(0, 0).shape == (0, ) @@ -46,8 +49,10 @@ def check_edge_case_of_sample_int(sample_without_replacement): assert sample_without_replacement(5, 1).shape == (1, ) # n_population < 0 or n_samples < 0 - assert_raises(ValueError, sample_without_replacement, -1, 5) - assert_raises(ValueError, sample_without_replacement, 5, -1) + with pytest.raises(ValueError): + sample_without_replacement(-1, 5) + with pytest.raises(ValueError): + sample_without_replacement(5, -1) def check_sample_int(sample_without_replacement): @@ -155,26 +160,26 @@ def test_random_choice_csc_errors(): # the length of an array in classes and class_probabilities is mismatched classes = [np.array([0, 1]), np.array([0, 1, 2, 3])] class_probabilities = [np.array([0.5, 0.5]), np.array([0.6, 0.1, 0.3])] - assert_raises(ValueError, _random_choice_csc, 4, classes, - class_probabilities, 1) + with pytest.raises(ValueError): + _random_choice_csc(4, classes, class_probabilities, 1) # the class dtype is not supported classes = [np.array(["a", "1"]), np.array(["z", "1", "2"])] class_probabilities = [np.array([0.5, 0.5]), np.array([0.6, 0.1, 0.3])] - assert_raises(ValueError, _random_choice_csc, 4, classes, - class_probabilities, 1) + with pytest.raises(ValueError): + _random_choice_csc(4, classes, class_probabilities, 1) # the class dtype is not supported classes = [np.array([4.2, 0.1]), np.array([0.1, 0.2, 9.4])] class_probabilities = [np.array([0.5, 0.5]), np.array([0.6, 0.1, 0.3])] - assert_raises(ValueError, _random_choice_csc, 4, classes, - class_probabilities, 1) + with pytest.raises(ValueError): + _random_choice_csc(4, classes, class_probabilities, 1) # Given probabilities don't sum to 1 classes = [np.array([0, 1]), np.array([0, 1, 2])] class_probabilities = [np.array([0.5, 0.6]), np.array([0.6, 0.1, 0.3])] - assert_raises(ValueError, _random_choice_csc, 4, classes, - class_probabilities, 1) + with pytest.raises(ValueError): + _random_choice_csc(4, classes, class_probabilities, 1) def test_our_rand_r(): diff --git a/sklearn/utils/tests/test_sparsefuncs.py b/sklearn/utils/tests/test_sparsefuncs.py index 92ff577bb70fc..ddb569f457249 100644 --- a/sklearn/utils/tests/test_sparsefuncs.py +++ b/sklearn/utils/tests/test_sparsefuncs.py @@ -16,8 +16,8 @@ count_nonzero, csc_median_axis_0) from sklearn.utils.sparsefuncs_fast import (assign_rows_csr, inplace_csr_row_normalize_l1, - inplace_csr_row_normalize_l2) -from sklearn.utils._testing import assert_raises + inplace_csr_row_normalize_l2, + csr_row_norms) from sklearn.utils._testing import assert_allclose @@ -31,7 +31,8 @@ def test_mean_variance_axis0(): X_lil[1, 0] = 0 X[1, 0] = 0 - assert_raises(TypeError, mean_variance_axis, X_lil, axis=0) + with pytest.raises(TypeError): + mean_variance_axis(X_lil, axis=0) X_csr = sp.csr_matrix(X_lil) X_csc = sp.csc_matrix(X_lil) @@ -62,7 +63,8 @@ def test_mean_variance_axis1(): X_lil[1, 0] = 0 X[1, 0] = 0 - assert_raises(TypeError, mean_variance_axis, X_lil, axis=1) + with pytest.raises(TypeError): + mean_variance_axis(X_lil, axis=1) X_csr = sp.csr_matrix(X_lil) X_csc = sp.csc_matrix(X_lil) @@ -101,12 +103,11 @@ def test_incr_mean_variance_axis(): X = np.atleast_2d(X) X_lil = sp.lil_matrix(X) X_csr = sp.csr_matrix(X_lil) - assert_raises(TypeError, incr_mean_variance_axis, axis, - last_mean, last_var, last_n) - assert_raises(TypeError, incr_mean_variance_axis, axis, - last_mean, last_var, last_n) - assert_raises(TypeError, incr_mean_variance_axis, X_lil, axis, - last_mean, last_var, last_n) + + with pytest.raises(TypeError): + incr_mean_variance_axis(axis, last_mean, last_var, last_n) + with pytest.raises(TypeError): + incr_mean_variance_axis(X_lil, axis, last_mean, last_var, last_n) # Test _incr_mean_and_var with a 1 row input X_means, X_vars = mean_variance_axis(X_csr, axis) @@ -150,6 +151,56 @@ def test_incr_mean_variance_axis(): assert_array_equal(X.shape[axis], n_incr) +@pytest.mark.parametrize( + "X1, X2", + [ + (sp.random(5, 2, density=0.8, format='csr', random_state=0), + sp.random(13, 2, density=0.8, format='csr', random_state=0)), + (sp.random(5, 2, density=0.8, format='csr', random_state=0), + sp.hstack([sp.csr_matrix(np.full((13, 1), fill_value=np.nan)), + sp.random(13, 1, density=0.8, random_state=42)], + format="csr")) + ] +) +def test_incr_mean_variance_axis_equivalence_mean_variance(X1, X2): + # non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/16448 + # check that computing the incremental mean and variance is equivalent to + # computing the mean and variance on the stacked dataset. + axis = 0 + last_mean, last_var = np.zeros(X1.shape[1]), np.zeros(X1.shape[1]) + last_n = np.zeros(X1.shape[1], dtype=np.int64) + updated_mean, updated_var, updated_n = incr_mean_variance_axis( + X1, axis, last_mean, last_var, last_n + ) + updated_mean, updated_var, updated_n = incr_mean_variance_axis( + X2, axis, updated_mean, updated_var, updated_n + ) + X = sp.vstack([X1, X2]) + assert_allclose(updated_mean, np.nanmean(X.A, axis=axis)) + assert_allclose(updated_var, np.nanvar(X.A, axis=axis)) + assert_allclose(updated_n, np.count_nonzero(~np.isnan(X.A), axis=0)) + + +def test_incr_mean_variance_no_new_n(): + # check the behaviour when we update the variance with an empty matrix + axis = 0 + X1 = sp.random(5, 1, density=0.8, random_state=0).tocsr() + X2 = sp.random(0, 1, density=0.8, random_state=0).tocsr() + last_mean, last_var = np.zeros(X1.shape[1]), np.zeros(X1.shape[1]) + last_n = np.zeros(X1.shape[1], dtype=np.int64) + last_mean, last_var, last_n = incr_mean_variance_axis( + X1, axis, last_mean, last_var, last_n + ) + # update statistic with a column which should ignored + updated_mean, updated_var, updated_n = incr_mean_variance_axis( + X2, axis, last_mean, last_var, last_n + ) + assert_allclose(updated_mean, last_mean) + assert_allclose(updated_var, last_var) + assert_allclose(updated_n, last_n) + + @pytest.mark.parametrize("axis", [0, 1]) @pytest.mark.parametrize("sparse_constructor", [sp.csc_matrix, sp.csr_matrix]) def test_incr_mean_variance_axis_ignore_nan(axis, sparse_constructor): @@ -194,16 +245,24 @@ def test_mean_variance_illegal_axis(): X[2, 1] = 0 X[4, 3] = 0 X_csr = sp.csr_matrix(X) - assert_raises(ValueError, mean_variance_axis, X_csr, axis=-3) - assert_raises(ValueError, mean_variance_axis, X_csr, axis=2) - assert_raises(ValueError, mean_variance_axis, X_csr, axis=-1) + with pytest.raises(ValueError): + mean_variance_axis(X_csr, axis=-3) + with pytest.raises(ValueError): + mean_variance_axis(X_csr, axis=2) + with pytest.raises(ValueError): + mean_variance_axis(X_csr, axis=-1) + + with pytest.raises(ValueError): + incr_mean_variance_axis(X_csr, axis=-3, last_mean=None, last_var=None, + last_n=None) - assert_raises(ValueError, incr_mean_variance_axis, X_csr, axis=-3, - last_mean=None, last_var=None, last_n=None) - assert_raises(ValueError, incr_mean_variance_axis, X_csr, axis=2, - last_mean=None, last_var=None, last_n=None) - assert_raises(ValueError, incr_mean_variance_axis, X_csr, axis=-1, - last_mean=None, last_var=None, last_n=None) + with pytest.raises(ValueError): + incr_mean_variance_axis(X_csr, axis=2, last_mean=None, last_var=None, + last_n=None) + + with pytest.raises(ValueError): + incr_mean_variance_axis(X_csr, axis=-1, last_mean=None, last_var=None, + last_n=None) def test_densify_rows(): @@ -238,7 +297,8 @@ def test_inplace_column_scale(): assert_array_almost_equal(Xr.toarray(), Xc.toarray()) assert_array_almost_equal(XA, Xc.toarray()) assert_array_almost_equal(XA, Xr.toarray()) - assert_raises(TypeError, inplace_column_scale, X.tolil(), scale) + with pytest.raises(TypeError): + inplace_column_scale(X.tolil(), scale) X = X.astype(np.float32) scale = scale.astype(np.float32) @@ -251,7 +311,8 @@ def test_inplace_column_scale(): assert_array_almost_equal(Xr.toarray(), Xc.toarray()) assert_array_almost_equal(XA, Xc.toarray()) assert_array_almost_equal(XA, Xr.toarray()) - assert_raises(TypeError, inplace_column_scale, X.tolil(), scale) + with pytest.raises(TypeError): + inplace_column_scale(X.tolil(), scale) def test_inplace_row_scale(): @@ -268,7 +329,8 @@ def test_inplace_row_scale(): assert_array_almost_equal(Xr.toarray(), Xc.toarray()) assert_array_almost_equal(XA, Xc.toarray()) assert_array_almost_equal(XA, Xr.toarray()) - assert_raises(TypeError, inplace_column_scale, X.tolil(), scale) + with pytest.raises(TypeError): + inplace_column_scale(X.tolil(), scale) X = X.astype(np.float32) scale = scale.astype(np.float32) @@ -281,7 +343,8 @@ def test_inplace_row_scale(): assert_array_almost_equal(Xr.toarray(), Xc.toarray()) assert_array_almost_equal(XA, Xc.toarray()) assert_array_almost_equal(XA, Xr.toarray()) - assert_raises(TypeError, inplace_column_scale, X.tolil(), scale) + with pytest.raises(TypeError): + inplace_column_scale(X.tolil(), scale) def test_inplace_swap_row(): @@ -308,7 +371,8 @@ def test_inplace_swap_row(): assert_array_equal(X_csr.toarray(), X_csc.toarray()) assert_array_equal(X, X_csc.toarray()) assert_array_equal(X, X_csr.toarray()) - assert_raises(TypeError, inplace_swap_row, X_csr.tolil()) + with pytest.raises(TypeError): + inplace_swap_row(X_csr.tolil()) X = np.array([[0, 3, 0], [2, 4, 0], @@ -331,7 +395,8 @@ def test_inplace_swap_row(): assert_array_equal(X_csr.toarray(), X_csc.toarray()) assert_array_equal(X, X_csc.toarray()) assert_array_equal(X, X_csr.toarray()) - assert_raises(TypeError, inplace_swap_row, X_csr.tolil()) + with pytest.raises(TypeError): + inplace_swap_row(X_csr.tolil()) def test_inplace_swap_column(): @@ -358,7 +423,8 @@ def test_inplace_swap_column(): assert_array_equal(X_csr.toarray(), X_csc.toarray()) assert_array_equal(X, X_csc.toarray()) assert_array_equal(X, X_csr.toarray()) - assert_raises(TypeError, inplace_swap_column, X_csr.tolil()) + with pytest.raises(TypeError): + inplace_swap_column(X_csr.tolil()) X = np.array([[0, 3, 0], [2, 4, 0], @@ -381,7 +447,8 @@ def test_inplace_swap_column(): assert_array_equal(X_csr.toarray(), X_csc.toarray()) assert_array_equal(X, X_csc.toarray()) assert_array_equal(X, X_csr.toarray()) - assert_raises(TypeError, inplace_swap_column, X_csr.tolil()) + with pytest.raises(TypeError): + inplace_swap_column(X_csr.tolil()) @pytest.mark.parametrize("dtype", [np.float32, np.float64]) @@ -419,9 +486,12 @@ def test_min_max_axis_errors(): [4, 0, 5]], dtype=np.float64) X_csr = sp.csr_matrix(X) X_csc = sp.csc_matrix(X) - assert_raises(TypeError, min_max_axis, X_csr.tolil(), axis=0) - assert_raises(ValueError, min_max_axis, X_csr, axis=2) - assert_raises(ValueError, min_max_axis, X_csc, axis=-3) + with pytest.raises(TypeError): + min_max_axis(X_csr.tolil(), axis=0) + with pytest.raises(ValueError): + min_max_axis(X_csr, axis=2) + with pytest.raises(ValueError): + min_max_axis(X_csc, axis=-3) def test_count_nonzero(): @@ -443,8 +513,10 @@ def test_count_nonzero(): sample_weight=sample_weight), X_nonzero_weighted.sum(axis=axis)) - assert_raises(TypeError, count_nonzero, X_csc) - assert_raises(ValueError, count_nonzero, X_csr, axis=2) + with pytest.raises(TypeError): + count_nonzero(X_csc) + with pytest.raises(ValueError): + count_nonzero(X_csr, axis=2) assert (count_nonzero(X_csr, axis=0).dtype == count_nonzero(X_csr, axis=1).dtype) @@ -452,7 +524,7 @@ def test_count_nonzero(): count_nonzero(X_csr, axis=1, sample_weight=sample_weight).dtype) # Check dtypes with large sparse matrices too - # XXX: test fails on Appveyor (python3.5 32bit) + # XXX: test fails on 32bit (Windows/Linux) try: X_csr.indices = X_csr.indices.astype(np.int64) X_csr.indptr = X_csr.indptr.astype(np.int64) @@ -463,11 +535,8 @@ def test_count_nonzero(): count_nonzero(X_csr, axis=1, sample_weight=sample_weight).dtype) except TypeError as e: - if ("according to the rule 'safe'" in e.args[0] and - np.intp().nbytes < 8): - pass - else: - raise + assert ("according to the rule 'safe'" in e.args[0] + and np.intp().nbytes < 8), e def test_csc_row_median(): @@ -500,7 +569,8 @@ def test_csc_row_median(): assert_array_equal(csc_median_axis_0(csc), np.array([0., -3])) # Test that it raises an Error for non-csc matrices. - assert_raises(TypeError, csc_median_axis_0, sp.csr_matrix(X)) + with pytest.raises(TypeError): + csc_median_axis_0(sp.csr_matrix(X)) def test_inplace_normalize(): @@ -525,3 +595,17 @@ def test_inplace_normalize(): if inplace_csr_row_normalize is inplace_csr_row_normalize_l2: X_csr.data **= 2 assert_array_almost_equal(np.abs(X_csr).sum(axis=1), ones) + + +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +def test_csr_row_norms(dtype): + # checks that csr_row_norms returns the same output as + # scipy.sparse.linalg.norm, and that the dype is the same as X.dtype. + X = sp.random(100, 10, format='csr', dtype=dtype, random_state=42) + + scipy_norms = sp.linalg.norm(X, axis=1)**2 + norms = csr_row_norms(X) + + assert norms.dtype == dtype + rtol = 1e-6 if dtype == np.float32 else 1e-7 + assert_allclose(norms, scipy_norms, rtol=rtol) diff --git a/sklearn/utils/tests/test_testing.py b/sklearn/utils/tests/test_testing.py index 64001bef9348b..4eafaad97fbb2 100644 --- a/sklearn/utils/tests/test_testing.py +++ b/sklearn/utils/tests/test_testing.py @@ -32,9 +32,9 @@ assert_raises_regex, TempMemmap, create_memmap_backed_data, - _delete_folder) + _delete_folder, + _convert_container) -from sklearn.utils._testing import SkipTest from sklearn.tree import DecisionTreeClassifier from sklearn.discriminant_analysis import LinearDiscriminantAnalysis @@ -43,14 +43,16 @@ category=FutureWarning) # 0.24 def test_assert_less(): assert 0 < 1 - assert_raises(AssertionError, assert_less, 1, 0) + with pytest.raises(AssertionError): + assert_less(1, 0) @pytest.mark.filterwarnings("ignore", category=FutureWarning) # 0.24 def test_assert_greater(): assert 1 > 0 - assert_raises(AssertionError, assert_greater, 0, 1) + with pytest.raises(AssertionError): + assert_greater(0, 1) @pytest.mark.filterwarnings("ignore", @@ -58,7 +60,8 @@ def test_assert_greater(): def test_assert_less_equal(): assert 0 <= 1 assert 1 <= 1 - assert_raises(AssertionError, assert_less_equal, 1, 0) + with pytest.raises(AssertionError): + assert_less_equal(1, 0) @pytest.mark.filterwarnings("ignore", @@ -66,7 +69,8 @@ def test_assert_less_equal(): def test_assert_greater_equal(): assert 1 >= 0 assert 1 >= 1 - assert_raises(AssertionError, assert_greater_equal, 0, 1) + with pytest.raises(AssertionError): + assert_greater_equal(0, 1) def test_set_random_state(): @@ -84,18 +88,17 @@ def test_assert_allclose_dense_sparse(): y = sparse.csc_matrix(x) for X in [x, y]: # basic compare - assert_raise_message(AssertionError, msg, assert_allclose_dense_sparse, - X, X * 2) + with pytest.raises(AssertionError, match=msg): + assert_allclose_dense_sparse(X, X*2) assert_allclose_dense_sparse(X, X) - assert_raise_message(ValueError, "Can only compare two sparse", - assert_allclose_dense_sparse, x, y) + with pytest.raises(ValueError, match="Can only compare two sparse"): + assert_allclose_dense_sparse(x, y) A = sparse.diags(np.ones(5), offsets=0).tocsr() B = sparse.csr_matrix(np.ones((1, 5))) - - assert_raise_message(AssertionError, "Arrays are not equal", - assert_allclose_dense_sparse, B, A) + with pytest.raises(AssertionError, match="Arrays are not equal"): + assert_allclose_dense_sparse(B, A) def test_assert_raises_msg(): @@ -251,8 +254,8 @@ def f(): # test that assert_warns doesn't have side effects on warnings # filters assert warnings.filters == filters_orig - - assert_raises(AssertionError, assert_no_warnings, f) + with pytest.raises(AssertionError): + assert_no_warnings(f) assert assert_no_warnings(lambda x: x, 1) == 1 def test_warn_wrong_warning(self): @@ -264,6 +267,9 @@ def f(): try: try: # Should raise an AssertionError + + # assert_warns has a special handling of "FutureWarning" that + # pytest.warns does not have assert_warns(UserWarning, f) failed = True except AssertionError: @@ -479,11 +485,8 @@ def fit(self, X, y): def test_check_docstring_parameters(): - try: - import numpydoc # noqa - except ImportError: - raise SkipTest( - "numpydoc is required to test the docstrings") + pytest.importorskip('numpydoc', + reason="numpydoc is required to test the docstrings") incorrect = check_docstring_parameters(f_ok) assert incorrect == [] @@ -491,10 +494,10 @@ def test_check_docstring_parameters(): assert incorrect == [] incorrect = check_docstring_parameters(f_missing, ignore=['b']) assert incorrect == [] - assert_raise_message(RuntimeError, 'Unknown section Results', - check_docstring_parameters, f_bad_sections) - assert_raise_message(RuntimeError, 'Unknown section Parameter', - check_docstring_parameters, Klass.f_bad_sections) + with pytest.raises(RuntimeError, match="Unknown section Results"): + check_docstring_parameters(f_bad_sections) + with pytest.raises(RuntimeError, match="Unknown section Parameter"): + check_docstring_parameters(Klass.f_bad_sections) incorrect = check_docstring_parameters(f_check_param_definition) assert ( @@ -674,3 +677,20 @@ def test_deprecated_helpers(callable, args): '0.24. Please use "assert" instead') with pytest.warns(FutureWarning, match=msg): callable(*args) + + +@pytest.mark.parametrize( + "constructor_name, container_type", + [('list', list), + ('tuple', tuple), + ('array', np.ndarray), + ('sparse', sparse.csr_matrix), + ('dataframe', pytest.importorskip('pandas').DataFrame), + ('series', pytest.importorskip('pandas').Series), + ('index', pytest.importorskip('pandas').Index), + ('slice', slice)] +) +def test_convert_container(constructor_name, container_type): + container = [0, 1] + assert isinstance(_convert_container(container, constructor_name), + container_type) diff --git a/sklearn/utils/tests/test_utils.py b/sklearn/utils/tests/test_utils.py index 2cf1e59a73f29..6103febd73616 100644 --- a/sklearn/utils/tests/test_utils.py +++ b/sklearn/utils/tests/test_utils.py @@ -8,14 +8,15 @@ import numpy as np import scipy.sparse as sp -from sklearn.utils._testing import (assert_raises, - assert_array_equal, - assert_allclose_dense_sparse, - assert_raises_regex, - assert_warns_message, assert_no_warnings) +from sklearn.utils._testing import (assert_array_equal, + assert_allclose_dense_sparse, + assert_warns_message, + assert_no_warnings, + _convert_container) from sklearn.utils import check_random_state from sklearn.utils import _determine_key_type from sklearn.utils import deprecated +from sklearn.utils import gen_batches from sklearn.utils import _get_column_indices from sklearn.utils import resample from sklearn.utils import safe_mask @@ -26,6 +27,7 @@ from sklearn.utils import _message_with_time, _print_elapsed_time from sklearn.utils import get_chunk_n_rows from sklearn.utils import is_scalar_nan +from sklearn.utils import _to_object_array from sklearn.utils._mocking import MockDataFrame from sklearn import config_context @@ -47,7 +49,24 @@ def test_make_rng(): rng_42 = np.random.RandomState(42) assert check_random_state(43).randint(100) != rng_42.randint(100) - assert_raises(ValueError, check_random_state, "some invalid seed") + with pytest.raises(ValueError): + check_random_state("some invalid seed") + + +def test_gen_batches(): + # Make sure gen_batches errors on invalid batch_size + + assert_array_equal( + list(gen_batches(4, 2)), + [slice(0, 2, None), slice(2, 4, None)] + ) + msg_zero = "gen_batches got batch_size=0, must be positive" + with pytest.raises(ValueError, match=msg_zero): + next(gen_batches(4, 0)) + + msg_float = "gen_batches got batch_size=0.5, must be an integer" + with pytest.raises(TypeError, match=msg_float): + next(gen_batches(4, 0.5)) def test_deprecated(): @@ -92,10 +111,13 @@ def test_resample(): assert resample() is None # Check that invalid arguments yield ValueError - assert_raises(ValueError, resample, [0], [0, 1]) - assert_raises(ValueError, resample, [0, 1], [0, 1], - replace=False, n_samples=3) - assert_raises(ValueError, resample, [0, 1], [0, 1], meaning_of_life=42) + with pytest.raises(ValueError): + resample([0], [0, 1]) + with pytest.raises(ValueError): + resample([0, 1], [0, 1], replace=False, n_samples=3) + + with pytest.raises(ValueError): + resample([0, 1], [0, 1], meaning_of_life=42) # Issue:6581, n_samples can be more when replace is True (default). assert len(resample([1, 2], n_samples=5)) == 5 @@ -194,7 +216,8 @@ def test_column_or_1d(): if y_type in ["binary", 'multiclass', "continuous"]: assert_array_equal(column_or_1d(y), np.ravel(y)) else: - assert_raises(ValueError, column_or_1d, y) + with pytest.raises(ValueError): + column_or_1d(y) @pytest.mark.parametrize( @@ -236,25 +259,6 @@ def test_determine_key_type_slice_error(): _determine_key_type(slice(0, 2, 1), accept_slice=False) -def _convert_container(container, constructor_name, columns_name=None): - if constructor_name == 'list': - return list(container) - elif constructor_name == 'tuple': - return tuple(container) - elif constructor_name == 'array': - return np.asarray(container) - elif constructor_name == 'sparse': - return sp.csr_matrix(container) - elif constructor_name == 'dataframe': - pd = pytest.importorskip('pandas') - return pd.DataFrame(container, columns=columns_name) - elif constructor_name == 'series': - pd = pytest.importorskip('pandas') - return pd.Series(container) - elif constructor_name == 'slice': - return slice(container[0], container[1]) - - @pytest.mark.parametrize( "array_type", ["list", "array", "sparse", "dataframe"] ) @@ -486,6 +490,22 @@ def test_get_column_indices_error(key, err_msg): _get_column_indices(X_df, key) +@pytest.mark.parametrize( + "key", + [['col1'], ['col2'], ['col1', 'col2'], ['col1', 'col3'], ['col2', 'col3']] +) +def test_get_column_indices_pandas_nonunique_columns_error(key): + pd = pytest.importorskip('pandas') + toy = np.zeros((1, 5), dtype=int) + columns = ['col1', 'col1', 'col2', 'col3', 'col2'] + X = pd.DataFrame(toy, columns=columns) + + err_msg = "Selected columns, {}, are not unique in dataframe".format(key) + with pytest.raises(ValueError) as exc_info: + _get_column_indices(X, key) + assert str(exc_info.value) == err_msg + + def test_shuffle_on_ndim_equals_three(): def to_tuple(A): # to make the inner arrays hashable return tuple(tuple(tuple(C) for C in B) for B in A) @@ -538,8 +558,9 @@ def test_gen_even_slices(): # check that passing negative n_chunks raises an error slices = gen_even_slices(10, -1) - assert_raises_regex(ValueError, "gen_even_slices got n_packs=-1, must be" - " >=1", next, slices) + with pytest.raises(ValueError, match="gen_even_slices got n_packs=-1," + " must be >=1"): + next(slices) @pytest.mark.parametrize( @@ -654,20 +675,6 @@ def dummy_func(): def test_deprecation_joblib_api(tmpdir): - def check_warning(*args, **kw): - return assert_warns_message( - FutureWarning, "deprecated in version 0.20.1", - *args, **kw) - - # Ensure that the joblib API is deprecated in sklearn.util - from sklearn.utils import Parallel, Memory, delayed - from sklearn.utils import cpu_count, hash, effective_n_jobs - check_warning(Memory, str(tmpdir)) - check_warning(hash, 1) - check_warning(Parallel) - check_warning(cpu_count) - check_warning(effective_n_jobs, 1) - check_warning(delayed, dummy_func) # Only parallel_backend and register_parallel_backend are not deprecated in # sklearn.utils @@ -675,19 +682,16 @@ def check_warning(*args, **kw): assert_no_warnings(parallel_backend, 'loky', None) assert_no_warnings(register_parallel_backend, 'failing', None) - # Ensure that the deprecation have no side effect in sklearn.utils._joblib - from sklearn.utils._joblib import Parallel, Memory, delayed - from sklearn.utils._joblib import cpu_count, hash, effective_n_jobs - from sklearn.utils._joblib import parallel_backend - from sklearn.utils._joblib import register_parallel_backend - assert_no_warnings(Memory, str(tmpdir)) - assert_no_warnings(hash, 1) - assert_no_warnings(Parallel) - assert_no_warnings(cpu_count) - assert_no_warnings(effective_n_jobs, 1) - assert_no_warnings(delayed, dummy_func) - assert_no_warnings(parallel_backend, 'loky', None) - assert_no_warnings(register_parallel_backend, 'failing', None) - from sklearn.utils._joblib import joblib del joblib.parallel.BACKENDS['failing'] + + +@pytest.mark.parametrize( + "sequence", + [[np.array(1), np.array(2)], [[1, 2], [3, 4]]] +) +def test_to_object_array(sequence): + out = _to_object_array(sequence) + assert isinstance(out, np.ndarray) + assert out.dtype.kind == 'O' + assert out.ndim == 1 diff --git a/sklearn/utils/tests/test_validation.py b/sklearn/utils/tests/test_validation.py index 1d71a3d7dd1e4..6748dbcad9951 100644 --- a/sklearn/utils/tests/test_validation.py +++ b/sklearn/utils/tests/test_validation.py @@ -5,17 +5,14 @@ from tempfile import NamedTemporaryFile from itertools import product +from operator import itemgetter import pytest from pytest import importorskip import numpy as np import scipy.sparse as sp -from sklearn.utils._testing import assert_raises -from sklearn.utils._testing import assert_raises_regex from sklearn.utils._testing import assert_no_warnings -from sklearn.utils._testing import assert_warns_message -from sklearn.utils._testing import assert_warns from sklearn.utils._testing import ignore_warnings from sklearn.utils._testing import SkipTest from sklearn.utils._testing import assert_array_equal @@ -32,6 +29,7 @@ from sklearn.ensemble import RandomForestRegressor from sklearn.svm import SVR from sklearn.datasets import make_blobs +from sklearn.utils import _safe_indexing from sklearn.utils.validation import ( has_fit_parameter, check_is_fitted, @@ -41,16 +39,17 @@ check_non_negative, _num_samples, check_scalar, + _check_psd_eigenvalues, _deprecate_positional_args, _check_sample_weight, _allclose_dense_sparse, FLOAT_DTYPES) +from sklearn.utils.validation import _check_fit_params + import sklearn -from sklearn.exceptions import NotFittedError -from sklearn.exceptions import DataConversionWarning +from sklearn.exceptions import NotFittedError, PositiveSpectrumWarning -from sklearn.utils._testing import assert_raise_message from sklearn.utils._testing import TempMemmap @@ -230,20 +229,26 @@ def test_check_array(): # raise error on sparse inputs X = [[1, 2], [3, 4]] X_csr = sp.csr_matrix(X) - assert_raises(TypeError, check_array, X_csr) + with pytest.raises(TypeError): + check_array(X_csr) + # ensure_2d=False X_array = check_array([0, 1, 2], ensure_2d=False) assert X_array.ndim == 1 # ensure_2d=True with 1d array - assert_raise_message(ValueError, 'Expected 2D array, got 1D array instead', - check_array, [0, 1, 2], ensure_2d=True) + with pytest.raises(ValueError, match="Expected 2D array," + " got 1D array instead"): + check_array([0, 1, 2], ensure_2d=True) + # ensure_2d=True with scalar array - assert_raise_message(ValueError, - 'Expected 2D array, got scalar array instead', - check_array, 10, ensure_2d=True) + with pytest.raises(ValueError, match="Expected 2D array," + " got scalar array instead"): + check_array(10, ensure_2d=True) + # don't allow ndim > 3 X_ndim = np.arange(8).reshape(2, 2, 2) - assert_raises(ValueError, check_array, X_ndim) + with pytest.raises(ValueError): + check_array(X_ndim) check_array(X_ndim, allow_nd=True) # doesn't raise # dtype and order enforcement. @@ -292,6 +297,7 @@ def test_check_array(): X_checked = check_array(X, dtype=dtype, accept_sparse=accept_sparse, copy=copy) if (dtype is object or sp.isspmatrix_dok(X)) and len(w): + # XXX unreached code as of v0.22 message = str(w[0].message) messages = ["object dtype is not supported by sparse matrices", "Can't check dok sparse matrix for nan or inf."] @@ -320,8 +326,10 @@ def test_check_array(): X_dense = check_array([[1, 2], [3, 4]]) assert isinstance(X_dense, np.ndarray) # raise on too deep lists - assert_raises(ValueError, check_array, X_ndim.tolist()) + with pytest.raises(ValueError): + check_array(X_ndim.tolist()) check_array(X_ndim.tolist(), allow_nd=True) # doesn't raise + # convert weird stuff to arrays X_no_array = _NotAnArray(X_dense) result = check_array(X_no_array) @@ -410,55 +418,18 @@ def test_check_array_dtype_stability(): def test_check_array_dtype_warning(): X_int_list = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] - X_float64 = np.asarray(X_int_list, dtype=np.float64) X_float32 = np.asarray(X_int_list, dtype=np.float32) X_int64 = np.asarray(X_int_list, dtype=np.int64) - X_csr_float64 = sp.csr_matrix(X_float64) X_csr_float32 = sp.csr_matrix(X_float32) X_csc_float32 = sp.csc_matrix(X_float32) X_csc_int32 = sp.csc_matrix(X_int64, dtype=np.int32) - y = [0, 0, 1] integer_data = [X_int64, X_csc_int32] - float64_data = [X_float64, X_csr_float64] float32_data = [X_float32, X_csr_float32, X_csc_float32] for X in integer_data: X_checked = assert_no_warnings(check_array, X, dtype=np.float64, accept_sparse=True) assert X_checked.dtype == np.float64 - X_checked = assert_warns(DataConversionWarning, check_array, X, - dtype=np.float64, - accept_sparse=True, warn_on_dtype=True) - assert X_checked.dtype == np.float64 - - # Check that the warning message includes the name of the Estimator - X_checked = assert_warns_message(DataConversionWarning, - 'SomeEstimator', - check_array, X, - dtype=[np.float64, np.float32], - accept_sparse=True, - warn_on_dtype=True, - estimator='SomeEstimator') - assert X_checked.dtype == np.float64 - - X_checked, y_checked = assert_warns_message( - DataConversionWarning, 'KNeighborsClassifier', - check_X_y, X, y, dtype=np.float64, accept_sparse=True, - warn_on_dtype=True, estimator=KNeighborsClassifier()) - - assert X_checked.dtype == np.float64 - - for X in float64_data: - with pytest.warns(None) as record: - warnings.simplefilter("ignore", FutureWarning) # 0.23 - X_checked = check_array(X, dtype=np.float64, - accept_sparse=True, warn_on_dtype=True) - assert X_checked.dtype == np.float64 - X_checked = check_array(X, dtype=np.float64, - accept_sparse=True, warn_on_dtype=False) - assert X_checked.dtype == np.float64 - assert len(record) == 0 - for X in float32_data: X_checked = assert_no_warnings(check_array, X, dtype=[np.float64, np.float32], @@ -482,41 +453,29 @@ def test_check_array_dtype_warning(): assert X_checked.format == 'csr' -def test_check_array_warn_on_dtype_deprecation(): - X = np.asarray([[0.0], [1.0]]) - Y = np.asarray([[2.0], [3.0]]) - with pytest.warns(FutureWarning, - match="'warn_on_dtype' is deprecated"): - check_array(X, warn_on_dtype=True) - with pytest.warns(FutureWarning, - match="'warn_on_dtype' is deprecated"): - check_X_y(X, Y, warn_on_dtype=True) - - def test_check_array_accept_sparse_type_exception(): X = [[1, 2], [3, 4]] X_csr = sp.csr_matrix(X) invalid_type = SVR() msg = ("A sparse matrix was passed, but dense data is required. " - "Use X.toarray() to convert to a dense numpy array.") - assert_raise_message(TypeError, msg, - check_array, X_csr, accept_sparse=False) + r"Use X.toarray\(\) to convert to a dense numpy array.") + with pytest.raises(TypeError, match=msg): + check_array(X_csr, accept_sparse=False) msg = ("Parameter 'accept_sparse' should be a string, " - "boolean or list of strings. You provided 'accept_sparse={}'.") - assert_raise_message(ValueError, msg.format(invalid_type), - check_array, X_csr, accept_sparse=invalid_type) + "boolean or list of strings. You provided 'accept_sparse=.*'.") + with pytest.raises(ValueError, match=msg): + check_array(X_csr, accept_sparse=invalid_type) msg = ("When providing 'accept_sparse' as a tuple or list, " "it must contain at least one string value.") - assert_raise_message(ValueError, msg.format([]), - check_array, X_csr, accept_sparse=[]) - assert_raise_message(ValueError, msg.format(()), - check_array, X_csr, accept_sparse=()) - - assert_raise_message(TypeError, "SVR", - check_array, X_csr, accept_sparse=[invalid_type]) + with pytest.raises(ValueError, match=msg): + check_array(X_csr, accept_sparse=[]) + with pytest.raises(ValueError, match=msg): + check_array(X_csr, accept_sparse=()) + with pytest.raises(TypeError, match="SVR"): + check_array(X_csr, accept_sparse=[invalid_type]) def test_check_array_accept_sparse_no_exception(): @@ -547,57 +506,64 @@ def test_check_array_accept_large_sparse_raise_exception(X_64bit): # When large sparse are not allowed msg = ("Only sparse matrices with 32-bit integer indices " "are accepted. Got int64 indices.") - assert_raise_message(ValueError, msg, - check_array, X_64bit, - accept_sparse=True, - accept_large_sparse=False) + with pytest.raises(ValueError, match=msg): + check_array(X_64bit, accept_sparse=True, accept_large_sparse=False) def test_check_array_min_samples_and_features_messages(): # empty list is considered 2D by default: - msg = "0 feature(s) (shape=(1, 0)) while a minimum of 1 is required." - assert_raise_message(ValueError, msg, check_array, [[]]) + msg = r"0 feature\(s\) \(shape=\(1, 0\)\) while a minimum of 1 is" \ + " required." + with pytest.raises(ValueError, match=msg): + check_array([[]]) # If considered a 1D collection when ensure_2d=False, then the minimum # number of samples will break: - msg = "0 sample(s) (shape=(0,)) while a minimum of 1 is required." - assert_raise_message(ValueError, msg, check_array, [], ensure_2d=False) + msg = r"0 sample\(s\) \(shape=\(0,\)\) while a minimum of 1 is required." + with pytest.raises(ValueError, match=msg): + check_array([], ensure_2d=False) # Invalid edge case when checking the default minimum sample of a scalar - msg = "Singleton array array(42) cannot be considered a valid collection." - assert_raise_message(TypeError, msg, check_array, 42, ensure_2d=False) + msg = r"Singleton array array\(42\) cannot be considered a valid" \ + " collection." + with pytest.raises(TypeError, match=msg): + check_array(42, ensure_2d=False) # Simulate a model that would need at least 2 samples to be well defined X = np.ones((1, 10)) y = np.ones(1) - msg = "1 sample(s) (shape=(1, 10)) while a minimum of 2 is required." - assert_raise_message(ValueError, msg, check_X_y, X, y, - ensure_min_samples=2) + msg = r"1 sample\(s\) \(shape=\(1, 10\)\) while a minimum of 2 is" \ + " required." + with pytest.raises(ValueError, match=msg): + check_X_y(X, y, ensure_min_samples=2) # The same message is raised if the data has 2 dimensions even if this is # not mandatory - assert_raise_message(ValueError, msg, check_X_y, X, y, - ensure_min_samples=2, ensure_2d=False) + with pytest.raises(ValueError, match=msg): + check_X_y(X, y, ensure_min_samples=2, ensure_2d=False) # Simulate a model that would require at least 3 features (e.g. SelectKBest # with k=3) X = np.ones((10, 2)) y = np.ones(2) - msg = "2 feature(s) (shape=(10, 2)) while a minimum of 3 is required." - assert_raise_message(ValueError, msg, check_X_y, X, y, - ensure_min_features=3) + msg = r"2 feature\(s\) \(shape=\(10, 2\)\) while a minimum of 3 is" \ + " required." + with pytest.raises(ValueError, match=msg): + check_X_y(X, y, ensure_min_features=3) # Only the feature check is enabled whenever the number of dimensions is 2 # even if allow_nd is enabled: - assert_raise_message(ValueError, msg, check_X_y, X, y, - ensure_min_features=3, allow_nd=True) + with pytest.raises(ValueError, match=msg): + check_X_y(X, y, ensure_min_features=3, allow_nd=True) # Simulate a case where a pipeline stage as trimmed all the features of a # 2D dataset. X = np.empty(0).reshape(10, 0) y = np.ones(10) - msg = "0 feature(s) (shape=(10, 0)) while a minimum of 1 is required." - assert_raise_message(ValueError, msg, check_X_y, X, y) + msg = r"0 feature\(s\) \(shape=\(10, 0\)\) while a minimum of 1 is" \ + " required." + with pytest.raises(ValueError, match=msg): + check_X_y(X, y) # nd-data is not checked for any minimum number of features by default: X = np.ones((10, 0, 28, 28)) @@ -609,41 +575,41 @@ def test_check_array_min_samples_and_features_messages(): def test_check_array_complex_data_error(): X = np.array([[1 + 2j, 3 + 4j, 5 + 7j], [2 + 3j, 4 + 5j, 6 + 7j]]) - assert_raises_regex( - ValueError, "Complex data not supported", check_array, X) + with pytest.raises(ValueError, match="Complex data not supported"): + check_array(X) # list of lists X = [[1 + 2j, 3 + 4j, 5 + 7j], [2 + 3j, 4 + 5j, 6 + 7j]] - assert_raises_regex( - ValueError, "Complex data not supported", check_array, X) + with pytest.raises(ValueError, match="Complex data not supported"): + check_array(X) # tuple of tuples X = ((1 + 2j, 3 + 4j, 5 + 7j), (2 + 3j, 4 + 5j, 6 + 7j)) - assert_raises_regex( - ValueError, "Complex data not supported", check_array, X) + with pytest.raises(ValueError, match="Complex data not supported"): + check_array(X) # list of np arrays X = [np.array([1 + 2j, 3 + 4j, 5 + 7j]), np.array([2 + 3j, 4 + 5j, 6 + 7j])] - assert_raises_regex( - ValueError, "Complex data not supported", check_array, X) + with pytest.raises(ValueError, match="Complex data not supported"): + check_array(X) # tuple of np arrays X = (np.array([1 + 2j, 3 + 4j, 5 + 7j]), np.array([2 + 3j, 4 + 5j, 6 + 7j])) - assert_raises_regex( - ValueError, "Complex data not supported", check_array, X) + with pytest.raises(ValueError, match="Complex data not supported"): + check_array(X) # dataframe X = MockDataFrame( np.array([[1 + 2j, 3 + 4j, 5 + 7j], [2 + 3j, 4 + 5j, 6 + 7j]])) - assert_raises_regex( - ValueError, "Complex data not supported", check_array, X) + with pytest.raises(ValueError, match="Complex data not supported"): + check_array(X) # sparse matrix X = sp.coo_matrix([[0, 1 + 2j], [0, 0]]) - assert_raises_regex( - ValueError, "Complex data not supported", check_array, X) + with pytest.raises(ValueError, match="Complex data not supported"): + check_array(X) def test_has_fit_parameter(): @@ -676,13 +642,16 @@ def test_check_symmetric(): 'bsr': sp.bsr_matrix(arr_asym)} # check error for bad inputs - assert_raises(ValueError, check_symmetric, arr_bad) + with pytest.raises(ValueError): + check_symmetric(arr_bad) # check that asymmetric arrays are properly symmetrized for arr_format, arr in test_arrays.items(): # Check for warnings and errors - assert_warns(UserWarning, check_symmetric, arr) - assert_raises(ValueError, check_symmetric, arr, raise_exception=True) + with pytest.warns(UserWarning): + check_symmetric(arr) + with pytest.raises(ValueError): + check_symmetric(arr, raise_exception=True) output = check_symmetric(arr, raise_warning=False) if sp.issparse(output): @@ -694,15 +663,19 @@ def test_check_symmetric(): def test_check_is_fitted(): # Check is TypeError raised when non estimator instance passed - assert_raises(TypeError, check_is_fitted, ARDRegression) - assert_raises(TypeError, check_is_fitted, "SVR") + with pytest.raises(TypeError): + check_is_fitted(ARDRegression) + with pytest.raises(TypeError): + check_is_fitted("SVR") ard = ARDRegression() svr = SVR() try: - assert_raises(NotFittedError, check_is_fitted, ard) - assert_raises(NotFittedError, check_is_fitted, svr) + with pytest.raises(NotFittedError): + check_is_fitted(ard) + with pytest.raises(NotFittedError): + check_is_fitted(svr) except ValueError: assert False, "check_is_fitted failed with ValueError" @@ -723,33 +696,70 @@ def test_check_is_fitted(): assert check_is_fitted(ard) is None assert check_is_fitted(svr) is None - # to be removed in 0.23 - assert_warns_message( - FutureWarning, - "Passing attributes to check_is_fitted is deprecated", - check_is_fitted, ard, ['coef_']) - assert_warns_message( - FutureWarning, - "Passing all_or_any to check_is_fitted is deprecated", - check_is_fitted, ard, all_or_any=any) + +def test_check_is_fitted_attributes(): + class MyEstimator(): + def fit(self, X, y): + return self + + msg = "not fitted" + est = MyEstimator() + + with pytest.raises(NotFittedError, match=msg): + check_is_fitted(est, attributes=["a_", "b_"]) + with pytest.raises(NotFittedError, match=msg): + check_is_fitted(est, attributes=["a_", "b_"], all_or_any=all) + with pytest.raises(NotFittedError, match=msg): + check_is_fitted(est, attributes=["a_", "b_"], all_or_any=any) + + est.a_ = "a" + with pytest.raises(NotFittedError, match=msg): + check_is_fitted(est, attributes=["a_", "b_"]) + with pytest.raises(NotFittedError, match=msg): + check_is_fitted(est, attributes=["a_", "b_"], all_or_any=all) + check_is_fitted(est, attributes=["a_", "b_"], all_or_any=any) + + est.b_ = "b" + check_is_fitted(est, attributes=["a_", "b_"]) + check_is_fitted(est, attributes=["a_", "b_"], all_or_any=all) + check_is_fitted(est, attributes=["a_", "b_"], all_or_any=any) + + +@pytest.mark.parametrize("wrap", + [itemgetter(0), list, tuple], + ids=["single", "list", "tuple"]) +def test_check_is_fitted_with_attributes(wrap): + ard = ARDRegression() + with pytest.raises(NotFittedError, match="is not fitted yet"): + check_is_fitted(ard, wrap(["coef_"])) + + ard.fit(*make_blobs()) + + # Does not raise + check_is_fitted(ard, wrap(["coef_"])) + + # Raises when using attribute that is not defined + with pytest.raises(NotFittedError, match="is not fitted yet"): + check_is_fitted(ard, wrap(["coef_bad_"])) def test_check_consistent_length(): check_consistent_length([1], [2], [3], [4], [5]) check_consistent_length([[1, 2], [[1, 2]]], [1, 2], ['a', 'b']) check_consistent_length([1], (2,), np.array([3]), sp.csr_matrix((1, 2))) - assert_raises_regex(ValueError, 'inconsistent numbers of samples', - check_consistent_length, [1, 2], [1]) - assert_raises_regex(TypeError, r"got <\w+ 'int'>", - check_consistent_length, [1, 2], 1) - assert_raises_regex(TypeError, r"got <\w+ 'object'>", - check_consistent_length, [1, 2], object()) - - assert_raises(TypeError, check_consistent_length, [1, 2], np.array(1)) + with pytest.raises(ValueError, match="inconsistent numbers of samples"): + check_consistent_length([1, 2], [1]) + with pytest.raises(TypeError, match=r"got <\w+ 'int'>"): + check_consistent_length([1, 2], 1) + with pytest.raises(TypeError, match=r"got <\w+ 'object'>"): + check_consistent_length([1, 2], object()) + + with pytest.raises(TypeError): + check_consistent_length([1, 2], np.array(1)) + # Despite ensembles having __len__ they must raise TypeError - assert_raises_regex(TypeError, 'Expected sequence or array-like', - check_consistent_length, [1, 2], - RandomForestRegressor()) + with pytest.raises(TypeError, match="Expected sequence or array-like"): + check_consistent_length([1, 2], RandomForestRegressor()) # XXX: We should have a test with a string, but what is correct behaviour? @@ -767,11 +777,13 @@ def test_check_dataframe_fit_attribute(): def test_suppress_validation(): X = np.array([0, np.inf]) - assert_raises(ValueError, assert_all_finite, X) + with pytest.raises(ValueError): + assert_all_finite(X) sklearn.set_config(assume_finite=True) assert_all_finite(X) sklearn.set_config(assume_finite=False) - assert_raises(ValueError, assert_all_finite, X) + with pytest.raises(ValueError): + assert_all_finite(X) def test_check_array_series(): @@ -786,42 +798,25 @@ def test_check_array_series(): assert_array_equal(res, np.array(['a', 'b', 'c'], dtype=object)) -def test_check_dataframe_warns_on_dtype(): - # Check that warn_on_dtype also works for DataFrames. - # https://github.com/scikit-learn/scikit-learn/issues/10948 - pd = importorskip("pandas") +def test_check_dataframe_mixed_float_dtypes(): + # pandas dataframe will coerce a boolean into a object, this is a mismatch + # with np.result_type which will return a float + # check_array needs to explicitly check for bool dtype in a dataframe for + # this situation + # https://github.com/scikit-learn/scikit-learn/issues/15787 - df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], dtype=object) - assert_warns_message(DataConversionWarning, - "Data with input dtype object were all converted to " - "float64.", - check_array, df, dtype=np.float64, warn_on_dtype=True) - assert_warns(DataConversionWarning, check_array, df, - dtype='numeric', warn_on_dtype=True) - with pytest.warns(None) as record: - warnings.simplefilter("ignore", FutureWarning) # 0.23 - check_array(df, dtype='object', warn_on_dtype=True) - assert len(record) == 0 + pd = importorskip("pandas") + df = pd.DataFrame({ + 'int': [1, 2, 3], + 'float': [0, 0.1, 2.1], + 'bool': [True, False, True]}, columns=['int', 'float', 'bool']) - # Also check that it raises a warning for mixed dtypes in a DataFrame. - df_mixed = pd.DataFrame([['1', 2, 3], ['4', 5, 6]]) - assert_warns(DataConversionWarning, check_array, df_mixed, - dtype=np.float64, warn_on_dtype=True) - assert_warns(DataConversionWarning, check_array, df_mixed, - dtype='numeric', warn_on_dtype=True) - assert_warns(DataConversionWarning, check_array, df_mixed, - dtype=object, warn_on_dtype=True) - - # Even with numerical dtypes, a conversion can be made because dtypes are - # uniformized throughout the array. - df_mixed_numeric = pd.DataFrame([[1., 2, 3], [4., 5, 6]]) - assert_warns(DataConversionWarning, check_array, df_mixed_numeric, - dtype='numeric', warn_on_dtype=True) - with pytest.warns(None) as record: - warnings.simplefilter("ignore", FutureWarning) # 0.23 - check_array(df_mixed_numeric.astype(int), - dtype='numeric', warn_on_dtype=True) - assert len(record) == 0 + array = check_array(df, dtype=(np.float64, np.float32, np.float16)) + expected_array = np.array( + [[1.0, 0.0, 1.0], + [2.0, 0.1, 0.0], + [3.0, 2.1, 1.0]], dtype=np.float) + assert_allclose_dense_sparse(array, expected_array) class DummyMemory: @@ -842,14 +837,16 @@ def test_check_memory(): dummy = DummyMemory() memory = check_memory(dummy) assert memory is dummy - assert_raises_regex(ValueError, "'memory' should be None, a string or" - " have the same interface as joblib.Memory." - " Got memory='1' instead.", check_memory, 1) + + msg = "'memory' should be None, a string or have the same interface as" \ + " joblib.Memory. Got memory='1' instead." + with pytest.raises(ValueError, match=msg): + check_memory(1) dummy = WrongDummyMemory() - assert_raises_regex(ValueError, "'memory' should be None, a string or" - " have the same interface as joblib.Memory." - " Got memory='{}' instead.".format(dummy), - check_memory, dummy) + msg = "'memory' should be None, a string or have the same interface as" \ + " joblib.Memory. Got memory='{}' instead.".format(dummy) + with pytest.raises(ValueError, match=msg): + check_memory(dummy) @pytest.mark.parametrize('copy', [True, False]) @@ -877,13 +874,15 @@ def test_check_non_negative(retype): A[0, 0] = -1 X = retype(A) - assert_raises_regex(ValueError, "Negative ", check_non_negative, X, "") + with pytest.raises(ValueError, match="Negative "): + check_non_negative(X, "") def test_check_X_y_informative_error(): X = np.ones((2, 2)) y = None - assert_raise_message(ValueError, "y cannot be None", check_X_y, X, y) + with pytest.raises(ValueError, match="y cannot be None"): + check_X_y(X, y) def test_retrieve_samples_from_non_standard_shape(): @@ -937,6 +936,81 @@ def test_check_scalar_invalid(x, target_name, target_type, min_val, max_val, assert type(raised_error.value) == type(err_msg) +_psd_cases_valid = { + 'nominal': ((1, 2), np.array([1, 2]), None, ""), + 'nominal_np_array': (np.array([1, 2]), np.array([1, 2]), None, ""), + 'insignificant_imag': ((5, 5e-5j), np.array([5, 0]), + PositiveSpectrumWarning, + "There are imaginary parts in eigenvalues " + "\\(1e\\-05 of the maximum real part"), + 'insignificant neg': ((5, -5e-5), np.array([5, 0]), + PositiveSpectrumWarning, ""), + 'insignificant neg float32': (np.array([1, -1e-6], dtype=np.float32), + np.array([1, 0], dtype=np.float32), + PositiveSpectrumWarning, + "There are negative eigenvalues \\(1e\\-06 " + "of the maximum positive"), + 'insignificant neg float64': (np.array([1, -1e-10], dtype=np.float64), + np.array([1, 0], dtype=np.float64), + PositiveSpectrumWarning, + "There are negative eigenvalues \\(1e\\-10 " + "of the maximum positive"), + 'insignificant pos': ((5, 4e-12), np.array([5, 0]), + PositiveSpectrumWarning, + "the largest eigenvalue is more than 1e\\+12 " + "times the smallest"), +} + + +@pytest.mark.parametrize("lambdas, expected_lambdas, w_type, w_msg", + list(_psd_cases_valid.values()), + ids=list(_psd_cases_valid.keys())) +@pytest.mark.parametrize("enable_warnings", [True, False]) +def test_check_psd_eigenvalues_valid(lambdas, expected_lambdas, w_type, w_msg, + enable_warnings): + # Test that ``_check_psd_eigenvalues`` returns the right output for valid + # input, possibly raising the right warning + + if not enable_warnings: + w_type = None + w_msg = "" + + with pytest.warns(w_type, match=w_msg) as w: + assert_array_equal( + _check_psd_eigenvalues(lambdas, enable_warnings=enable_warnings), + expected_lambdas + ) + if w_type is None: + assert not w + + +_psd_cases_invalid = { + 'significant_imag': ((5, 5j), ValueError, + "There are significant imaginary parts in eigenv"), + 'all negative': ((-5, -1), ValueError, + "All eigenvalues are negative \\(maximum is -1"), + 'significant neg': ((5, -1), ValueError, + "There are significant negative eigenvalues"), + 'significant neg float32': (np.array([3e-4, -2e-6], dtype=np.float32), + ValueError, + "There are significant negative eigenvalues"), + 'significant neg float64': (np.array([1e-5, -2e-10], dtype=np.float64), + ValueError, + "There are significant negative eigenvalues"), +} + + +@pytest.mark.parametrize("lambdas, err_type, err_msg", + list(_psd_cases_invalid.values()), + ids=list(_psd_cases_invalid.keys())) +def test_check_psd_eigenvalues_invalid(lambdas, err_type, err_msg): + # Test that ``_check_psd_eigenvalues`` raises the right error for invalid + # input + + with pytest.raises(err_type, match=err_msg): + _check_psd_eigenvalues(lambdas) + + def test_check_sample_weight(): # check array order sample_weight = np.ones(10)[::2] @@ -1051,3 +1125,31 @@ def __init__(self, a=1, b=1, *, c=1, d=1): with pytest.warns(FutureWarning, match=r"Pass c=3, d=4 as keyword args"): A2(1, 2, 3, 4) + + +@pytest.mark.parametrize("indices", [None, [1, 3]]) +def test_check_fit_params(indices): + X = np.random.randn(4, 2) + fit_params = { + 'list': [1, 2, 3, 4], + 'array': np.array([1, 2, 3, 4]), + 'sparse-col': sp.csc_matrix([1, 2, 3, 4]).T, + 'sparse-row': sp.csc_matrix([1, 2, 3, 4]), + 'scalar-int': 1, + 'scalar-str': 'xxx', + 'None': None, + } + result = _check_fit_params(X, fit_params, indices) + indices_ = indices if indices is not None else list(range(X.shape[0])) + + for key in ['sparse-row', 'scalar-int', 'scalar-str', 'None']: + assert result[key] is fit_params[key] + + assert result['list'] == _safe_indexing(fit_params['list'], indices_) + assert_array_equal( + result['array'], _safe_indexing(fit_params['array'], indices_) + ) + assert_allclose_dense_sparse( + result['sparse-col'], + _safe_indexing(fit_params['sparse-col'], indices_) + ) diff --git a/sklearn/utils/validation.py b/sklearn/utils/validation.py index dad56850f2235..08952d6cbcd16 100644 --- a/sklearn/utils/validation.py +++ b/sklearn/utils/validation.py @@ -6,6 +6,7 @@ # Lars Buitinck # Alexandre Gramfort # Nicolas Tresegnie +# Sylvain Marie # License: BSD 3 clause from functools import wraps @@ -20,9 +21,11 @@ from numpy.core.numeric import ComplexWarning import joblib +from contextlib import suppress + from .fixes import _object_dtype_isnan from .. import get_config as _get_config -from ..exceptions import NonBLASDotWarning +from ..exceptions import NonBLASDotWarning, PositiveSpectrumWarning from ..exceptions import NotFittedError from ..exceptions import DataConversionWarning @@ -211,6 +214,26 @@ def check_consistent_length(*arrays): " samples: %r" % [int(l) for l in lengths]) +def _make_indexable(iterable): + """Ensure iterable supports indexing or convert to an indexable variant. + + Convert sparse matrices to csr and other non-indexable iterable to arrays. + Let `None` and indexable objects (e.g. pandas dataframes) pass unchanged. + + Parameters + ---------- + iterable : {list, dataframe, array, sparse} or None + Object to be converted to an indexable iterable. + """ + if sp.issparse(iterable): + return iterable.tocsr() + elif hasattr(iterable, "__getitem__") or hasattr(iterable, "iloc"): + return iterable + elif iterable is None: + return iterable + return np.array(iterable) + + def indexable(*iterables): """Make arrays indexable for cross-validation. @@ -223,16 +246,7 @@ def indexable(*iterables): *iterables : lists, dataframes, arrays, sparse matrices List of objects to ensure sliceability. """ - result = [] - for X in iterables: - if sp.issparse(X): - result.append(X.tocsr()) - elif hasattr(X, "__getitem__") or hasattr(X, "iloc"): - result.append(X) - elif X is None: - result.append(X) - else: - result.append(np.array(X)) + result = [_make_indexable(X) for X in iterables] check_consistent_length(*result) return result @@ -338,7 +352,7 @@ def _ensure_no_complex_data(array): def check_array(array, accept_sparse=False, accept_large_sparse=True, dtype="numeric", order=None, copy=False, force_all_finite=True, ensure_2d=True, allow_nd=False, ensure_min_samples=1, - ensure_min_features=1, warn_on_dtype=None, estimator=None): + ensure_min_features=1, estimator=None): """Input validation on an array, list, sparse matrix or similar. @@ -413,14 +427,6 @@ def check_array(array, accept_sparse=False, accept_large_sparse=True, dimensions or is originally 1D and ``ensure_2d`` is True. Setting to 0 disables this check. - warn_on_dtype : boolean or None, optional (default=None) - Raise DataConversionWarning if the dtype of the input data structure - does not match the requested dtype, causing a memory copy. - - .. deprecated:: 0.21 - ``warn_on_dtype`` is deprecated in version 0.21 and will be - removed in 0.23. - estimator : str or estimator instance (default=None) If passed, include the name of the estimator in warning messages. @@ -429,14 +435,6 @@ def check_array(array, accept_sparse=False, accept_large_sparse=True, array_converted : object The converted and validated array. """ - # warn_on_dtype deprecation - if warn_on_dtype is not None: - warnings.warn( - "'warn_on_dtype' is deprecated in version 0.21 and will be " - "removed in 0.23. Don't set `warn_on_dtype` to remove this " - "warning.", - FutureWarning, stacklevel=2) - # store reference to original array to check if copy is needed when # function returns array_orig = array @@ -453,9 +451,23 @@ def check_array(array, accept_sparse=False, accept_large_sparse=True, # DataFrame), and store them. If not, store None. dtypes_orig = None if hasattr(array, "dtypes") and hasattr(array.dtypes, '__array__'): - dtypes_orig = np.array(array.dtypes) + # throw warning if pandas dataframe is sparse + with suppress(ImportError): + from pandas.api.types import is_sparse + if array.dtypes.apply(is_sparse).any(): + warnings.warn( + "pandas.DataFrame with sparse columns found." + "It will be converted to a dense numpy array." + ) + + dtypes_orig = list(array.dtypes) + # pandas boolean dtype __array__ interface coerces bools to objects + for i, dtype_iter in enumerate(dtypes_orig): + if dtype_iter.kind == 'b': + dtypes_orig[i] = np.object + if all(isinstance(dtype, np.dtype) for dtype in dtypes_orig): - dtype_orig = np.result_type(*array.dtypes) + dtype_orig = np.result_type(*dtypes_orig) if dtype_numeric: if dtype_orig is not None and dtype_orig.kind == "O": @@ -576,24 +588,9 @@ def check_array(array, accept_sparse=False, accept_large_sparse=True, % (n_features, array.shape, ensure_min_features, context)) - if warn_on_dtype and dtype_orig is not None and array.dtype != dtype_orig: - msg = ("Data with input dtype %s was converted to %s%s." - % (dtype_orig, array.dtype, context)) - warnings.warn(msg, DataConversionWarning, stacklevel=2) - if copy and np.may_share_memory(array, array_orig): array = np.array(array, dtype=dtype, order=order) - if (warn_on_dtype and dtypes_orig is not None and - {array.dtype} != set(dtypes_orig)): - # if there was at the beginning some other types than the final one - # (for instance in a DataFrame that can contain several dtypes) then - # some data must have been converted - msg = ("Data with input dtype %s were all converted to %s%s." - % (', '.join(map(str, sorted(set(dtypes_orig)))), array.dtype, - context)) - warnings.warn(msg, DataConversionWarning, stacklevel=3) - return array @@ -620,7 +617,7 @@ def check_X_y(X, y, accept_sparse=False, accept_large_sparse=True, dtype="numeric", order=None, copy=False, force_all_finite=True, ensure_2d=True, allow_nd=False, multi_output=False, ensure_min_samples=1, ensure_min_features=1, y_numeric=False, - warn_on_dtype=None, estimator=None): + estimator=None): """Input validation for standard estimators. Checks X and y for consistent length, enforces X to be 2D and y 1D. By @@ -705,14 +702,6 @@ def check_X_y(X, y, accept_sparse=False, accept_large_sparse=True, it is converted to float64. Should only be used for regression algorithms. - warn_on_dtype : boolean or None, optional (default=None) - Raise DataConversionWarning if the dtype of the input data structure - does not match the requested dtype, causing a memory copy. - - .. deprecated:: 0.21 - ``warn_on_dtype`` is deprecated in version 0.21 and will be - removed in 0.23. - estimator : str or estimator instance (default=None) If passed, include the name of the estimator in warning messages. @@ -734,7 +723,6 @@ def check_X_y(X, y, accept_sparse=False, accept_large_sparse=True, ensure_2d=ensure_2d, allow_nd=allow_nd, ensure_min_samples=ensure_min_samples, ensure_min_features=ensure_min_features, - warn_on_dtype=warn_on_dtype, estimator=estimator) if multi_output: y = check_array(y, 'csr', force_all_finite=True, ensure_2d=False, @@ -777,7 +765,9 @@ def column_or_1d(y, warn=False): DataConversionWarning, stacklevel=2) return np.ravel(y) - raise ValueError("bad input shape {0}".format(shape)) + raise ValueError( + "y should be a 1d array, " + "got an array of shape {} instead.".format(shape)) def check_random_state(seed): @@ -884,23 +874,28 @@ def check_symmetric(array, tol=1E-10, raise_warning=True, return array -def check_is_fitted(estimator, attributes='deprecated', msg=None, - all_or_any='deprecated'): +def check_is_fitted(estimator, attributes=None, msg=None, all_or_any=all): """Perform is_fitted validation for estimator. Checks if the estimator is fitted by verifying the presence of fitted attributes (ending with a trailing underscore) and otherwise raises a NotFittedError with the given message. + This utility is meant to be used internally by estimators themselves, + typically in their own predict / transform methods. + Parameters ---------- estimator : estimator instance. estimator instance for which the check is performed. - attributes : deprecated, ignored - .. deprecated:: 0.22 - `attributes` is deprecated, is currently ignored and will be removed - in 0.23. + attributes : str, list or tuple of str, default=None + Attribute name(s) given as string or a list/tuple of strings + Eg.: ``["coef_", "estimator_", ...], "coef_"`` + + If `None`, `estimator` is considered fitted if there exist an + attribute that ends with a underscore and does not start with double + underscore. msg : string The default error message is, "This %(name)s instance is not fitted @@ -912,10 +907,8 @@ def check_is_fitted(estimator, attributes='deprecated', msg=None, Eg. : "Estimator, %(name)s, must be fitted before sparsifying". - all_or_any : deprecated, ignored - .. deprecated:: 0.21 - `all_or_any` is deprecated, is currently ignored and will be removed - in 0.23. + all_or_any : callable, {all, any}, default all + Specify whether all or any of the given attributes must exist. Returns ------- @@ -926,14 +919,6 @@ def check_is_fitted(estimator, attributes='deprecated', msg=None, NotFittedError If the attributes are not found. """ - if attributes != 'deprecated': - warnings.warn("Passing attributes to check_is_fitted is deprecated" - " and will be removed in 0.23. The attributes " - "argument is ignored.", FutureWarning) - if all_or_any != 'deprecated': - warnings.warn("Passing all_or_any to check_is_fitted is deprecated" - " and will be removed in 0.23. The any_or_all " - "argument is ignored.", FutureWarning) if isclass(estimator): raise TypeError("{} is a class, not an instance.".format(estimator)) if msg is None: @@ -943,9 +928,13 @@ def check_is_fitted(estimator, attributes='deprecated', msg=None, if not hasattr(estimator, 'fit'): raise TypeError("%s is not an estimator instance." % (estimator)) - attrs = [v for v in vars(estimator) - if (v.endswith("_") or v.startswith("_")) - and not v.startswith("__")] + if attributes is not None: + if not isinstance(attributes, (list, tuple)): + attributes = [attributes] + attrs = all_or_any([hasattr(estimator, attr) for attr in attributes]) + else: + attrs = [v for v in vars(estimator) + if v.endswith("_") and not v.startswith("__")] if not attrs: raise NotFittedError(msg % {'name': type(estimator).__name__}) @@ -1020,6 +1009,169 @@ def check_scalar(x, name, target_type, min_val=None, max_val=None): raise ValueError('`{}`= {}, must be <= {}.'.format(name, x, max_val)) +def _check_psd_eigenvalues(lambdas, enable_warnings=False): + """Check the eigenvalues of a positive semidefinite (PSD) matrix. + + Checks the provided array of PSD matrix eigenvalues for numerical or + conditioning issues and returns a fixed validated version. This method + should typically be used if the PSD matrix is user-provided (e.g. a + Gram matrix) or computed using a user-provided dissimilarity metric + (e.g. kernel function), or if the decomposition process uses approximation + methods (randomized SVD, etc.). + + It checks for three things: + + - that there are no significant imaginary parts in eigenvalues (more than + 1e-5 times the maximum real part). If this check fails, it raises a + ``ValueError``. Otherwise all non-significant imaginary parts that may + remain are set to zero. This operation is traced with a + ``PositiveSpectrumWarning`` when ``enable_warnings=True``. + + - that eigenvalues are not all negative. If this check fails, it raises a + ``ValueError`` + + - that there are no significant negative eigenvalues with absolute value + more than 1e-10 (1e-6) and more than 1e-5 (5e-3) times the largest + positive eigenvalue in double (simple) precision. If this check fails, + it raises a ``ValueError``. Otherwise all negative eigenvalues that may + remain are set to zero. This operation is traced with a + ``PositiveSpectrumWarning`` when ``enable_warnings=True``. + + Finally, all the positive eigenvalues that are too small (with a value + smaller than the maximum eigenvalue divided by 1e12) are set to zero. + This operation is traced with a ``PositiveSpectrumWarning`` when + ``enable_warnings=True``. + + Parameters + ---------- + lambdas : array-like of shape (n_eigenvalues,) + Array of eigenvalues to check / fix. + + enable_warnings : bool, default=False + When this is set to ``True``, a ``PositiveSpectrumWarning`` will be + raised when there are imaginary parts, negative eigenvalues, or + extremely small non-zero eigenvalues. Otherwise no warning will be + raised. In both cases, imaginary parts, negative eigenvalues, and + extremely small non-zero eigenvalues will be set to zero. + + Returns + ------- + lambdas_fixed : ndarray of shape (n_eigenvalues,) + A fixed validated copy of the array of eigenvalues. + + Examples + -------- + >>> _check_psd_eigenvalues([1, 2]) # nominal case + array([1, 2]) + >>> _check_psd_eigenvalues([5, 5j]) # significant imag part + Traceback (most recent call last): + ... + ValueError: There are significant imaginary parts in eigenvalues (1 + of the maximum real part). Either the matrix is not PSD, or there was + an issue while computing the eigendecomposition of the matrix. + >>> _check_psd_eigenvalues([5, 5e-5j]) # insignificant imag part + array([5., 0.]) + >>> _check_psd_eigenvalues([-5, -1]) # all negative + Traceback (most recent call last): + ... + ValueError: All eigenvalues are negative (maximum is -1). Either the + matrix is not PSD, or there was an issue while computing the + eigendecomposition of the matrix. + >>> _check_psd_eigenvalues([5, -1]) # significant negative + Traceback (most recent call last): + ... + ValueError: There are significant negative eigenvalues (0.2 of the + maximum positive). Either the matrix is not PSD, or there was an issue + while computing the eigendecomposition of the matrix. + >>> _check_psd_eigenvalues([5, -5e-5]) # insignificant negative + array([5., 0.]) + >>> _check_psd_eigenvalues([5, 4e-12]) # bad conditioning (too small) + array([5., 0.]) + + """ + + lambdas = np.array(lambdas) + is_double_precision = lambdas.dtype == np.float64 + + # note: the minimum value available is + # - single-precision: np.finfo('float32').eps = 1.2e-07 + # - double-precision: np.finfo('float64').eps = 2.2e-16 + + # the various thresholds used for validation + # we may wish to change the value according to precision. + significant_imag_ratio = 1e-5 + significant_neg_ratio = 1e-5 if is_double_precision else 5e-3 + significant_neg_value = 1e-10 if is_double_precision else 1e-6 + small_pos_ratio = 1e-12 + + # Check that there are no significant imaginary parts + if not np.isreal(lambdas).all(): + max_imag_abs = np.abs(np.imag(lambdas)).max() + max_real_abs = np.abs(np.real(lambdas)).max() + if max_imag_abs > significant_imag_ratio * max_real_abs: + raise ValueError( + "There are significant imaginary parts in eigenvalues (%g " + "of the maximum real part). Either the matrix is not PSD, or " + "there was an issue while computing the eigendecomposition " + "of the matrix." + % (max_imag_abs / max_real_abs)) + + # warn about imaginary parts being removed + if enable_warnings: + warnings.warn("There are imaginary parts in eigenvalues (%g " + "of the maximum real part). Either the matrix is not" + " PSD, or there was an issue while computing the " + "eigendecomposition of the matrix. Only the real " + "parts will be kept." + % (max_imag_abs / max_real_abs), + PositiveSpectrumWarning) + + # Remove all imaginary parts (even if zero) + lambdas = np.real(lambdas) + + # Check that there are no significant negative eigenvalues + max_eig = lambdas.max() + if max_eig < 0: + raise ValueError("All eigenvalues are negative (maximum is %g). " + "Either the matrix is not PSD, or there was an " + "issue while computing the eigendecomposition of " + "the matrix." % max_eig) + + else: + min_eig = lambdas.min() + if (min_eig < -significant_neg_ratio * max_eig + and min_eig < -significant_neg_value): + raise ValueError("There are significant negative eigenvalues (%g" + " of the maximum positive). Either the matrix is " + "not PSD, or there was an issue while computing " + "the eigendecomposition of the matrix." + % (-min_eig / max_eig)) + elif min_eig < 0: + # Remove all negative values and warn about it + if enable_warnings: + warnings.warn("There are negative eigenvalues (%g of the " + "maximum positive). Either the matrix is not " + "PSD, or there was an issue while computing the" + " eigendecomposition of the matrix. Negative " + "eigenvalues will be replaced with 0." + % (-min_eig / max_eig), + PositiveSpectrumWarning) + lambdas[lambdas < 0] = 0 + + # Check for conditioning (small positive non-zeros) + too_small_lambdas = (0 < lambdas) & (lambdas < small_pos_ratio * max_eig) + if too_small_lambdas.any(): + if enable_warnings: + warnings.warn("Badly conditioned PSD matrix spectrum: the largest " + "eigenvalue is more than %g times the smallest. " + "Small eigenvalues will be replaced with 0." + "" % (1 / small_pos_ratio), + PositiveSpectrumWarning) + lambdas[too_small_lambdas] = 0 + + return lambdas + + def _check_sample_weight(sample_weight, X, dtype=None): """Validate sample weights. @@ -1053,12 +1205,10 @@ def _check_sample_weight(sample_weight, X, dtype=None): if dtype is not None and dtype not in [np.float32, np.float64]: dtype = np.float64 - if sample_weight is None or isinstance(sample_weight, numbers.Number): - if sample_weight is None: - sample_weight = np.ones(n_samples, dtype=dtype) - else: - sample_weight = np.full(n_samples, sample_weight, - dtype=dtype) + if sample_weight is None: + sample_weight = np.ones(n_samples, dtype=dtype) + elif isinstance(sample_weight, numbers.Number): + sample_weight = np.full(n_samples, sample_weight, dtype=dtype) else: if dtype is None: dtype = [np.float64, np.float32] @@ -1140,10 +1290,48 @@ def inner_f(*args, **kwargs): args_msg = ['{}={}'.format(name, arg) for name, arg in zip(kwonly_args[:extra_args], args[-extra_args:])] - warnings.warn("Pass {} as keyword args. From version 0.24 " + warnings.warn("Pass {} as keyword args. From version 0.25 " "passing these as positional arguments will " "result in an error".format(", ".join(args_msg)), FutureWarning) kwargs.update({k: arg for k, arg in zip(all_args, args)}) return f(**kwargs) return inner_f + + +def _check_fit_params(X, fit_params, indices=None): + """Check and validate the parameters passed during `fit`. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Data array. + + fit_params : dict + Dictionary containing the parameters passed at fit. + + indices : array-like of shape (n_samples,), default=None + Indices to be selected if the parameter has the same size as `X`. + + Returns + ------- + fit_params_validated : dict + Validated parameters. We ensure that the values support indexing. + """ + from . import _safe_indexing + fit_params_validated = {} + for param_key, param_value in fit_params.items(): + if (not _is_arraylike(param_value) or + _num_samples(param_value) != _num_samples(X)): + # Non-indexable pass-through (for now for backward-compatibility). + # https://github.com/scikit-learn/scikit-learn/issues/15805 + fit_params_validated[param_key] = param_value + else: + # Any other fit_params should support indexing + # (e.g. for cross-validation). + fit_params_validated[param_key] = _make_indexable(param_value) + fit_params_validated[param_key] = _safe_indexing( + fit_params_validated[param_key], indices + ) + + return fit_params_validated